From 6f9dabe49c9661e745a793411816d4d546833777 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 18 Jul 2017 20:42:55 +0200 Subject: [PATCH 001/305] unittests: fix version test. --- test/unit/utVersion.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/utVersion.cpp b/test/unit/utVersion.cpp index 2fce13b61..bbb82268f 100644 --- a/test/unit/utVersion.cpp +++ b/test/unit/utVersion.cpp @@ -54,11 +54,11 @@ TEST_F( utVersion, aiGetLegalStringTest ) { } TEST_F( utVersion, aiGetVersionMinorTest ) { - EXPECT_EQ( aiGetVersionMinor(), 3U ); + EXPECT_EQ( aiGetVersionMinor(), 4U ); } TEST_F( utVersion, aiGetVersionMajorTest ) { - EXPECT_EQ( aiGetVersionMajor(), 3U ); + EXPECT_EQ( aiGetVersionMajor(), 0U ); } TEST_F( utVersion, aiGetCompileFlagsTest ) { From b394546e55302c056431679de255bc84b72267ef Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 18 Jul 2017 20:53:47 +0200 Subject: [PATCH 002/305] unittests: fix typo. --- test/unit/utVersion.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/utVersion.cpp b/test/unit/utVersion.cpp index bbb82268f..677a131c9 100644 --- a/test/unit/utVersion.cpp +++ b/test/unit/utVersion.cpp @@ -54,11 +54,11 @@ TEST_F( utVersion, aiGetLegalStringTest ) { } TEST_F( utVersion, aiGetVersionMinorTest ) { - EXPECT_EQ( aiGetVersionMinor(), 4U ); + EXPECT_EQ( aiGetVersionMinor(), 0U ); } TEST_F( utVersion, aiGetVersionMajorTest ) { - EXPECT_EQ( aiGetVersionMajor(), 0U ); + EXPECT_EQ( aiGetVersionMajor(), 4U ); } TEST_F( utVersion, aiGetCompileFlagsTest ) { From a965414ceb4680f504e3b24adb1fd86918d2a1f5 Mon Sep 17 00:00:00 2001 From: Sergio Acereda Date: Wed, 19 Jul 2017 14:05:54 +0200 Subject: [PATCH 003/305] not compiling when using ASSIMP_DOUBLE_PRECISION --- include/assimp/cimport.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/assimp/cimport.h b/include/assimp/cimport.h index 98be269d3..8aa125c67 100644 --- a/include/assimp/cimport.h +++ b/include/assimp/cimport.h @@ -417,7 +417,7 @@ ASSIMP_API void aiSetImportPropertyInteger( ASSIMP_API void aiSetImportPropertyFloat( C_STRUCT aiPropertyStore* store, const char* szName, - float value); + ai_real value); // -------------------------------------------------------------------------------- /** Set a string property. From 3d4b54f8fcdd26308f20595ae7928a2d0470d771 Mon Sep 17 00:00:00 2001 From: rickomax Date: Wed, 19 Jul 2017 19:00:53 -0300 Subject: [PATCH 004/305] Fixed FBX 7500 Binary reading Seems that all FBX 7.5 Binary files uses 32 bits adresses. The code now is taking this in consideration. This commit fixes the https://github.com/assimp/assimp/issues/838 issue. --- code/FBXBinaryTokenizer.cpp | 113 ++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/code/FBXBinaryTokenizer.cpp b/code/FBXBinaryTokenizer.cpp index a02458fa5..fb57b2f97 100644 --- a/code/FBXBinaryTokenizer.cpp +++ b/code/FBXBinaryTokenizer.cpp @@ -56,47 +56,46 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { namespace FBX { -enum Flag -{ - e_unknown_0 = 1 << 0, - e_unknown_1 = 1 << 1, - e_unknown_2 = 1 << 2, - e_unknown_3 = 1 << 3, - e_unknown_4 = 1 << 4, - e_unknown_5 = 1 << 5, - e_unknown_6 = 1 << 6, - e_unknown_7 = 1 << 7, - e_unknown_8 = 1 << 8, - e_unknown_9 = 1 << 9, - e_unknown_10 = 1 << 10, - e_unknown_11 = 1 << 11, - e_unknown_12 = 1 << 12, - e_unknown_13 = 1 << 13, - e_unknown_14 = 1 << 14, - e_unknown_15 = 1 << 15, - e_unknown_16 = 1 << 16, - e_unknown_17 = 1 << 17, - e_unknown_18 = 1 << 18, - e_unknown_19 = 1 << 19, - e_unknown_20 = 1 << 20, - e_unknown_21 = 1 << 21, - e_unknown_22 = 1 << 22, - e_unknown_23 = 1 << 23, - e_flag_field_size_64_bit = 1 << 24, // Not sure what is - e_unknown_25 = 1 << 25, - e_unknown_26 = 1 << 26, - e_unknown_27 = 1 << 27, - e_unknown_28 = 1 << 28, - e_unknown_29 = 1 << 29, - e_unknown_30 = 1 << 30, - e_unknown_31 = 1 << 31 -}; - -bool check_flag(uint32_t flags, Flag to_check) -{ - return (flags & to_check) != 0; -} - +//enum Flag +//{ +// e_unknown_0 = 1 << 0, +// e_unknown_1 = 1 << 1, +// e_unknown_2 = 1 << 2, +// e_unknown_3 = 1 << 3, +// e_unknown_4 = 1 << 4, +// e_unknown_5 = 1 << 5, +// e_unknown_6 = 1 << 6, +// e_unknown_7 = 1 << 7, +// e_unknown_8 = 1 << 8, +// e_unknown_9 = 1 << 9, +// e_unknown_10 = 1 << 10, +// e_unknown_11 = 1 << 11, +// e_unknown_12 = 1 << 12, +// e_unknown_13 = 1 << 13, +// e_unknown_14 = 1 << 14, +// e_unknown_15 = 1 << 15, +// e_unknown_16 = 1 << 16, +// e_unknown_17 = 1 << 17, +// e_unknown_18 = 1 << 18, +// e_unknown_19 = 1 << 19, +// e_unknown_20 = 1 << 20, +// e_unknown_21 = 1 << 21, +// e_unknown_22 = 1 << 22, +// e_unknown_23 = 1 << 23, +// e_flag_field_size_64_bit = 1 << 24, // Not sure what is +// e_unknown_25 = 1 << 25, +// e_unknown_26 = 1 << 26, +// e_unknown_27 = 1 << 27, +// e_unknown_28 = 1 << 28, +// e_unknown_29 = 1 << 29, +// e_unknown_30 = 1 << 30, +// e_unknown_31 = 1 << 31 +//}; +// +//bool check_flag(uint32_t flags, Flag to_check) +//{ +// return (flags & to_check) != 0; +//} // ------------------------------------------------------------------------------------------------ Token::Token(const char* sbegin, const char* send, TokenType type, unsigned int offset) : @@ -341,10 +340,11 @@ void ReadData(const char*& sbegin_out, const char*& send_out, const char* input, // ------------------------------------------------------------------------------------------------ -bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, const char* end, uint32_t const flags) +//TODO: Test FBX Binary files newer than the 7500 version to check if the 64 bits address behaviour is consistent +bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, const char* end, uint32_t const version) { // the first word contains the offset at which this block ends - const uint64_t end_offset = /*check_flag(flags, e_flag_field_size_64_bit) ? ReadDoubleWord(input, cursor, end) : */ReadWord(input, cursor, end); + const uint64_t end_offset = version == 7500 ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); // we may get 0 if reading reached the end of the file - // fbx files have a mysterious extra footer which I don't know @@ -362,10 +362,10 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, } // the second data word contains the number of properties in the scope - const uint64_t prop_count = /*check_flag(flags, e_flag_field_size_64_bit) ? ReadDoubleWord(input, cursor, end) : */ReadWord(input, cursor, end); + const uint64_t prop_count = version == 7500 ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); // the third data word contains the length of the property list - const uint64_t prop_length = /*check_flag(flags, e_flag_field_size_64_bit) ? ReadDoubleWord(input, cursor, end) :*/ ReadWord(input, cursor, end); + const uint64_t prop_length = version == 7500 ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); // now comes the name of the scope/key const char* sbeg, *send; @@ -392,7 +392,7 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, // at the end of each nested block, there is a NUL record to indicate // that the sub-scope exists (i.e. to distinguish between P: and P : {}) // this NUL record is 13 bytes long on 32 bit version and 25 bytes long on 64 bit. - const size_t sentinel_block_length = /*check_flag(flags, e_flag_field_size_64_bit) ? (sizeof(uint64_t) * 3 + 1) : */(sizeof(uint32_t) * 3 + 1); + const size_t sentinel_block_length = version == 7500 ? (sizeof(uint64_t)* 3 + 1) : (sizeof(uint32_t)* 3 + 1); if (Offset(input, cursor) < end_offset) { if (end_offset - Offset(input, cursor) < sentinel_block_length) { @@ -403,7 +403,7 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, // XXX this is vulnerable to stack overflowing .. while(Offset(input, cursor) < end_offset - sentinel_block_length) { - ReadScope(output_tokens, input, cursor, input + end_offset - sentinel_block_length, flags); + ReadScope(output_tokens, input, cursor, input + end_offset - sentinel_block_length, version); } output_tokens.push_back(new_Token(cursor, cursor + 1, TokenType_CLOSE_BRACKET, Offset(input, cursor) )); @@ -438,19 +438,16 @@ void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int le TokenizeError("magic bytes not found",0); } - - //uint32_t offset = 0x15; - const char* cursor = input + 0x15; - - const uint32_t flags = ReadWord(input, cursor, input + length); - - const uint8_t padding_0 = ReadByte(input, cursor, input + length); // unused - (void) padding_0; - const uint8_t padding_1 = ReadByte(input, cursor, input + length); // unused - (void) padding_1; + const char* cursor = input + 18; + const uint8_t unknown_1 = ReadByte(input, cursor, input + length); + const uint8_t unknown_2 = ReadByte(input, cursor, input + length); + const uint8_t unknown_3 = ReadByte(input, cursor, input + length); + const uint8_t unknown_4 = ReadByte(input, cursor, input + length); + const uint8_t unknown_5 = ReadByte(input, cursor, input + length); + const uint32_t version = ReadWord(input, cursor, input + length); while (cursor < input + length) { - if(!ReadScope(output_tokens, input, cursor, input + length, flags)) { + if (!ReadScope(output_tokens, input, cursor, input + length, version)) { break; } } From 147541ab7f8eeffd2828530703cd8dfe792e8248 Mon Sep 17 00:00:00 2001 From: rickomax Date: Wed, 19 Jul 2017 19:04:10 -0300 Subject: [PATCH 005/305] Complementing last fix Complementing last fix --- code/FBXBinaryTokenizer.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/code/FBXBinaryTokenizer.cpp b/code/FBXBinaryTokenizer.cpp index fb57b2f97..d388c6446 100644 --- a/code/FBXBinaryTokenizer.cpp +++ b/code/FBXBinaryTokenizer.cpp @@ -340,11 +340,10 @@ void ReadData(const char*& sbegin_out, const char*& send_out, const char* input, // ------------------------------------------------------------------------------------------------ -//TODO: Test FBX Binary files newer than the 7500 version to check if the 64 bits address behaviour is consistent -bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, const char* end, uint32_t const version) +bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, const char* end, bool const is64bits) { // the first word contains the offset at which this block ends - const uint64_t end_offset = version == 7500 ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); + const uint64_t end_offset = is64bits ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); // we may get 0 if reading reached the end of the file - // fbx files have a mysterious extra footer which I don't know @@ -362,10 +361,10 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, } // the second data word contains the number of properties in the scope - const uint64_t prop_count = version == 7500 ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); + const uint64_t prop_count = is64bits ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); // the third data word contains the length of the property list - const uint64_t prop_length = version == 7500 ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); + const uint64_t prop_length = is64bits ? ReadDoubleWord(input, cursor, end) : ReadWord(input, cursor, end); // now comes the name of the scope/key const char* sbeg, *send; @@ -392,7 +391,7 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, // at the end of each nested block, there is a NUL record to indicate // that the sub-scope exists (i.e. to distinguish between P: and P : {}) // this NUL record is 13 bytes long on 32 bit version and 25 bytes long on 64 bit. - const size_t sentinel_block_length = version == 7500 ? (sizeof(uint64_t)* 3 + 1) : (sizeof(uint32_t)* 3 + 1); + const size_t sentinel_block_length = is64bits ? (sizeof(uint64_t)* 3 + 1) : (sizeof(uint32_t)* 3 + 1); if (Offset(input, cursor) < end_offset) { if (end_offset - Offset(input, cursor) < sentinel_block_length) { @@ -403,7 +402,7 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, // XXX this is vulnerable to stack overflowing .. while(Offset(input, cursor) < end_offset - sentinel_block_length) { - ReadScope(output_tokens, input, cursor, input + end_offset - sentinel_block_length, version); + ReadScope(output_tokens, input, cursor, input + end_offset - sentinel_block_length, is64bits); } output_tokens.push_back(new_Token(cursor, cursor + 1, TokenType_CLOSE_BRACKET, Offset(input, cursor) )); @@ -426,6 +425,7 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, } // ------------------------------------------------------------------------------------------------ +// TODO: Test FBX Binary files newer than the 7500 version to check if the 64 bits address behaviour is consistent void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int length) { ai_assert(input); @@ -445,9 +445,10 @@ void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int le const uint8_t unknown_4 = ReadByte(input, cursor, input + length); const uint8_t unknown_5 = ReadByte(input, cursor, input + length); const uint32_t version = ReadWord(input, cursor, input + length); + const bool is64bits = version == 7500; while (cursor < input + length) { - if (!ReadScope(output_tokens, input, cursor, input + length, version)) { + if (!ReadScope(output_tokens, input, cursor, input + length, is64bits)) { break; } } From 01a288ff35ea48fea9e8986e8e352d49a8b26924 Mon Sep 17 00:00:00 2001 From: rodrigo figueroa Date: Thu, 20 Jul 2017 22:48:07 -0700 Subject: [PATCH 006/305] Added support for python3 When using python3 and running any of the provided scripts I get: NameError: name 'unicode' is not defined. Python3 does not have unicode. Using try-except method helps to support python3 --- port/PyAssimp/pyassimp/core.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/port/PyAssimp/pyassimp/core.py b/port/PyAssimp/pyassimp/core.py index ac3df9290..573ce27b1 100644 --- a/port/PyAssimp/pyassimp/core.py +++ b/port/PyAssimp/pyassimp/core.py @@ -119,7 +119,10 @@ def _init(self, target = None, parent = None): if m == 'mName': obj = self.mName - uni = unicode(obj.data, errors='ignore') + try: + uni = unicode(obj.data, errors='ignore') + except: + uni = str(obj.data, errors='ignore') target.name = str( uni ) target.__class__.__repr__ = lambda x: str(x.__class__) + "(" + x.name + ")" target.__class__.__str__ = lambda x: x.name @@ -440,7 +443,10 @@ def _get_properties(properties, length): for p in [properties[i] for i in range(length)]: #the name p = p.contents - uni = unicode(p.mKey.data, errors='ignore') + try: + uni = unicode(p.mKey.data, errors='ignore') + except: + uni = str(p.mKey.data, errors='ignore') key = (str(uni).split('.')[1], p.mSemantic) #the data @@ -449,7 +455,10 @@ def _get_properties(properties, length): arr = cast(p.mData, POINTER(c_float * int(p.mDataLength/sizeof(c_float)) )).contents value = [x for x in arr] elif p.mType == 3: #string can't be an array - uni = unicode(cast(p.mData, POINTER(structs.MaterialPropertyString)).contents.data, errors='ignore') + try: + uni = unicode(cast(p.mData, POINTER(structs.MaterialPropertyString)).contents.data, errors='ignore') + except: + uni = str(cast(p.mData, POINTER(structs.MaterialPropertyString)).contents.data, errors='ignore') value = uni elif p.mType == 4: @@ -476,3 +485,4 @@ def decompose_matrix(matrix): from ctypes import byref, pointer _assimp_lib.dll.aiDecomposeMatrix(pointer(matrix), byref(scaling), byref(rotation), byref(position)) return scaling._init(), rotation._init(), position._init() + From f7a7e57807819619be7e674e065d11cb5a4e81cc Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 12:03:09 +0200 Subject: [PATCH 007/305] travis: check if cmake is installed with brew. --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 661de2279..918f6ad9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,14 @@ language: cpp before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install cmake && sudo apt-get install cmake python3 && sudo apt-get install -qq freeglut3-dev libxmu-dev libxi-dev ; echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install cmake python3 homebrew/x11/freeglut; fi + - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then + if brew ls versions cmake > /dev/null/ then + else + brew install cmake + fi + brew install python3 + brew install homebrew/x11/freeglut; + fi' - echo -e "#ifndef A_R_H_INC\n#define A_R_H_INC\n#define GitVersion ${TRAVIS_JOB_ID}\n#define GitBranch \"${TRAVIS_BRANCH}\"\n#endif // A_R_H_INC" > revision.h # install latest LCOV (1.9 was failing) - if [ "$TRAVIS_OS_NAME" = "linux" ]; then cd ${TRAVIS_BUILD_DIR} && wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.11.orig.tar.gz && tar xf lcov_1.11.orig.tar.gz && sudo make -C lcov-1.11/ install && gem install coveralls-lcov && lcov --version && g++ --version ; fi From 91f888a254821bb2e878c7e765e45b7427b5bbbc Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 12:36:10 +0200 Subject: [PATCH 008/305] travis: add missing semicolons. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 918f6ad9c..961104395 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,9 @@ before_install: - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then if brew ls versions cmake > /dev/null/ then else - brew install cmake + brew install cmake; fi - brew install python3 + brew install python3; brew install homebrew/x11/freeglut; fi' - echo -e "#ifndef A_R_H_INC\n#define A_R_H_INC\n#define GitVersion ${TRAVIS_JOB_ID}\n#define GitBranch \"${TRAVIS_BRANCH}\"\n#endif // A_R_H_INC" > revision.h From 11e9173a094f7810bcf17554b2c1b048607e4663 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 13:21:31 +0200 Subject: [PATCH 009/305] travis: add missing semicolon in if-then statement. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 961104395..452c14121 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: cpp before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install cmake && sudo apt-get install cmake python3 && sudo apt-get install -qq freeglut3-dev libxmu-dev libxi-dev ; echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then - if brew ls versions cmake > /dev/null/ then + if brew ls versions cmake > /dev/null/; then else brew install cmake; fi From e7afc8baf35d5255907b669a73ee40192f514780 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 14:37:42 +0200 Subject: [PATCH 010/305] travis: add next missing selicolon. --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 452c14121..28fc680f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,11 @@ before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install cmake && sudo apt-get install cmake python3 && sudo apt-get install -qq freeglut3-dev libxmu-dev libxi-dev ; echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then if brew ls versions cmake > /dev/null/; then - else + echo cmake already installed.; + else brew install cmake; - fi - brew install python3; + fi + brew install python3; brew install homebrew/x11/freeglut; fi' - echo -e "#ifndef A_R_H_INC\n#define A_R_H_INC\n#define GitVersion ${TRAVIS_JOB_ID}\n#define GitBranch \"${TRAVIS_BRANCH}\"\n#endif // A_R_H_INC" > revision.h From 501f09307ce69b849063cd3615e19498465eda94 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 18:55:25 +0200 Subject: [PATCH 011/305] travis: fix brew ls command. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 28fc680f7..071da1a49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: cpp before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install cmake && sudo apt-get install cmake python3 && sudo apt-get install -qq freeglut3-dev libxmu-dev libxi-dev ; echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then - if brew ls versions cmake > /dev/null/; then + if brew ls --versions cmake > /dev/null/; then echo cmake already installed.; else brew install cmake; From e94bbe7e0a19d2a75224df71aba06d379fc6c437 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 18:58:55 +0200 Subject: [PATCH 012/305] travis: fix dev/null access. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 071da1a49..e43e3a12d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: cpp before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install cmake && sudo apt-get install cmake python3 && sudo apt-get install -qq freeglut3-dev libxmu-dev libxi-dev ; echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then - if brew ls --versions cmake > /dev/null/; then + if brew ls --versions cmake > /dev/null; then echo cmake already installed.; else brew install cmake; From d150874f068f132d13c098eb72d78afc5a5cdc1b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 21 Jul 2017 19:10:32 +0200 Subject: [PATCH 013/305] travis: next try. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e43e3a12d..ffebab3cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ before_install: echo cmake already installed.; else brew install cmake; - fi + fi; brew install python3; brew install homebrew/x11/freeglut; fi' From 8606ec6060e599fac72cb17ff21f626665c53af3 Mon Sep 17 00:00:00 2001 From: Sergio Acereda Date: Sat, 22 Jul 2017 23:28:55 +0200 Subject: [PATCH 014/305] Low performance in OptimizeMeshesProcess::ProcessNode with huge number of aiMeshes --- code/OptimizeMeshes.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/code/OptimizeMeshes.cpp b/code/OptimizeMeshes.cpp index 65a3cadce..f8183fc75 100644 --- a/code/OptimizeMeshes.cpp +++ b/code/OptimizeMeshes.cpp @@ -181,11 +181,8 @@ void OptimizeMeshesProcess::ProcessNode( aiNode* pNode) verts += mScene->mMeshes[am]->mNumVertices; faces += mScene->mMeshes[am]->mNumFaces; + pNode->mMeshes[a] = pNode->mMeshes[pNode->mNumMeshes - 1]; --pNode->mNumMeshes; - for( unsigned int n = a; n < pNode->mNumMeshes; ++n ) { - pNode->mMeshes[ n ] = pNode->mMeshes[ n + 1 ]; - } - --a; } } From 6429502fc50ad57de2454320eafbb84348553270 Mon Sep 17 00:00:00 2001 From: Sergio Acereda Date: Sat, 22 Jul 2017 23:30:36 +0200 Subject: [PATCH 015/305] Elapsed seconds not shown correctly --- code/Profiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/Profiler.h b/code/Profiler.h index 40d436b52..96e898e6e 100644 --- a/code/Profiler.h +++ b/code/Profiler.h @@ -84,7 +84,7 @@ public: return; } - auto elapsedSeconds = std::chrono::system_clock::now() - regions[region]; + std::chrono::duration elapsedSeconds = std::chrono::system_clock::now() - regions[region]; DefaultLogger::get()->debug((format("END `"),region,"`, dt= ", elapsedSeconds.count()," s")); } From 2cb1a9efc631b964416a98bd1da8775b42b7f687 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 25 Jul 2017 00:00:43 +0200 Subject: [PATCH 016/305] Profiler: add a unittest. --- code/Profiler.h | 20 +++++----- include/assimp/LogStream.hpp | 18 +++++++-- test/CMakeLists.txt | 1 + test/unit/UnitTestPCH.h | 64 ++++++++++++++++++++++++++++-- test/unit/utProfiler.cpp | 77 ++++++++++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 test/unit/utProfiler.cpp diff --git a/code/Profiler.h b/code/Profiler.h index 96e898e6e..00cb5a9b6 100644 --- a/code/Profiler.h +++ b/code/Profiler.h @@ -52,21 +52,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include namespace Assimp { - namespace Profiling { - - using namespace Formatter; +namespace Profiling { +using namespace Formatter; // ------------------------------------------------------------------------------------------------ /** Simple wrapper around boost::timer to simplify reporting. Timings are automatically * dumped to the log file. */ -class Profiler -{ - +class Profiler { public: - - Profiler() {} + Profiler() { + // empty + } public: @@ -89,12 +87,12 @@ public: } private: - - typedef std::map> RegionMap; + using RegionMap = std::map>; RegionMap regions; }; - } +} } #endif + diff --git a/include/assimp/LogStream.hpp b/include/assimp/LogStream.hpp index 049640ec0..1052f1fda 100644 --- a/include/assimp/LogStream.hpp +++ b/include/assimp/LogStream.hpp @@ -64,12 +64,11 @@ class ASSIMP_API LogStream { protected: /** @brief Default constructor */ - LogStream() { - } + LogStream(); + public: /** @brief Virtual destructor */ - virtual ~LogStream() { - } + virtual ~LogStream(); // ------------------------------------------------------------------- /** @brief Overwrite this for your own output methods @@ -94,6 +93,17 @@ public: IOSystem* io = NULL); }; // !class LogStream + +inline +LogStream::LogStream() { + // empty +} + +inline +LogStream::~LogStream() { + // empty +} + // ------------------------------------------------------------------------------------ } // Namespace Assimp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b0be474d8..1d5b4ae5d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -129,6 +129,7 @@ SET( TEST_SRCS unit/utVector3.cpp unit/utXImporterExporter.cpp unit/utD3MFImportExport.cpp + unit/utProfiler.cpp ) SOURCE_GROUP( tests FILES ${TEST_SRCS} ) diff --git a/test/unit/UnitTestPCH.h b/test/unit/UnitTestPCH.h index 890bedd0c..96b05bd81 100644 --- a/test/unit/UnitTestPCH.h +++ b/test/unit/UnitTestPCH.h @@ -1,8 +1,45 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team -// #ifndef ASSIMP_BUILD_SINGLETHREADED -// # include -// #endif +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#pragma once // We need to be sure to have the same STL settings as Assimp @@ -10,6 +47,27 @@ #include #include #include +#include + +class UTLogStream : public Assimp::LogStream { +public: + UTLogStream() + : LogStream() { + // empty + } + + virtual ~UTLogStream() { + // empty + } + + void write(const char* message) override { + if ( nullptr != message ) { + m_messages.push_back( std::string( message ) ); + } + } + + std::vector m_messages; +}; #undef min #undef max diff --git a/test/unit/utProfiler.cpp b/test/unit/utProfiler.cpp new file mode 100644 index 000000000..b25167f9e --- /dev/null +++ b/test/unit/utProfiler.cpp @@ -0,0 +1,77 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#include "UnitTestPCH.h" + +#include "code/Profiler.h" +#include + +using namespace ::Assimp; +using namespace ::Assimp::Profiling; + +class utProfiler : public ::testing::Test { +public: + LogStream *m_stream; + + virtual void SetUp() { + m_stream = new UTLogStream; + DefaultLogger::create(); + DefaultLogger::get()->attachStream( m_stream ); + } + + virtual void TearDown() { + DefaultLogger::get()->detatchStream( m_stream ); + m_stream = nullptr; + DefaultLogger::kill(); + } +}; + +TEST_F( utProfiler, addRegion_success ) { + Profiler myProfiler; + myProfiler.BeginRegion( "t1" ); + for ( int i=0; i<10; i++ ) { + volatile int j=0; + j++; + } + myProfiler.EndRegion( "t1" ); + UTLogStream *stream( (UTLogStream*) m_stream ); + //EXPECT_FALSE( stream->m_messages.empty() ); +} From 4dc2cd8442d6996fd99cc9713a3375c866448478 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 25 Jul 2017 08:54:29 +0200 Subject: [PATCH 017/305] Profiler: fix invalid using statement. --- code/Profiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/Profiler.h b/code/Profiler.h index 00cb5a9b6..1c9ca60b5 100644 --- a/code/Profiler.h +++ b/code/Profiler.h @@ -87,7 +87,7 @@ public: } private: - using RegionMap = std::map>; + typedef std::map> RegionMap; RegionMap regions; }; From 256f28432a468ffed50b50d82e7bc2ab0aa2ce1f Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 25 Jul 2017 09:37:28 +0200 Subject: [PATCH 018/305] unittests: fix the build again. --- test/CMakeLists.txt | 1 + test/unit/UTLogStream.h | 63 +++++++++++++++++++++++++++++++++++++++++ test/unit/UnitTestPCH.h | 23 +-------------- 3 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 test/unit/UTLogStream.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1d5b4ae5d..2d065e3be 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -57,6 +57,7 @@ SOURCE_GROUP( unit FILES ) SET( TEST_SRCS + unit/UTLogStream.h unit/AbstractImportExportBase.cpp unit/TestIOSystem.h unit/TestModelFactory.h diff --git a/test/unit/UTLogStream.h b/test/unit/UTLogStream.h new file mode 100644 index 000000000..df4467b3e --- /dev/null +++ b/test/unit/UTLogStream.h @@ -0,0 +1,63 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#pragma once + +#include + +class UTLogStream : public Assimp::LogStream { +public: + UTLogStream() + : LogStream() { + // empty + } + + virtual ~UTLogStream() { + // empty + } + + void write(const char* message) override { + if ( nullptr != message ) { + m_messages.push_back( std::string( message ) ); + } + } + + std::vector m_messages; +}; diff --git a/test/unit/UnitTestPCH.h b/test/unit/UnitTestPCH.h index 96b05bd81..0d5f08992 100644 --- a/test/unit/UnitTestPCH.h +++ b/test/unit/UnitTestPCH.h @@ -5,7 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2017, assimp team - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -47,27 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include - -class UTLogStream : public Assimp::LogStream { -public: - UTLogStream() - : LogStream() { - // empty - } - - virtual ~UTLogStream() { - // empty - } - - void write(const char* message) override { - if ( nullptr != message ) { - m_messages.push_back( std::string( message ) ); - } - } - - std::vector m_messages; -}; +#include "UTLogStream.h" #undef min #undef max From 4013b319af8bccfa522b08aea10fdca10ae10a49 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 25 Jul 2017 09:38:35 +0200 Subject: [PATCH 019/305] unittests: add missing include to utProfiler. --- test/unit/utProfiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/utProfiler.cpp b/test/unit/utProfiler.cpp index b25167f9e..0edd08255 100644 --- a/test/unit/utProfiler.cpp +++ b/test/unit/utProfiler.cpp @@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------- */ #include "UnitTestPCH.h" - +#include "UTLogStream.h" #include "code/Profiler.h" #include From cf0a83de175a16f043eecf2a3e470aad2975f9f1 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 25 Jul 2017 09:46:54 +0200 Subject: [PATCH 020/305] unittests: fix build. --- test/unit/UTLogStream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/UTLogStream.h b/test/unit/UTLogStream.h index df4467b3e..87ebea91d 100644 --- a/test/unit/UTLogStream.h +++ b/test/unit/UTLogStream.h @@ -53,7 +53,7 @@ public: // empty } - void write(const char* message) override { + virtual void write(const char* message) { if ( nullptr != message ) { m_messages.push_back( std::string( message ) ); } From 38626d426097e2917238f9ee6d67c173037ffdcc Mon Sep 17 00:00:00 2001 From: jamesgk Date: Wed, 19 Jul 2017 13:21:43 -0700 Subject: [PATCH 021/305] glTF: start fork of files used in export, for glTF2 --- code/glTF2Asset.h | 1194 +++++++++++++++++++++++++++ code/glTF2Asset.inl | 1636 +++++++++++++++++++++++++++++++++++++ code/glTF2AssetWriter.h | 95 +++ code/glTF2AssetWriter.inl | 685 ++++++++++++++++ code/glTF2Exporter.cpp | 1007 +++++++++++++++++++++++ code/glTF2Exporter.h | 118 +++ 6 files changed, 4735 insertions(+) create mode 100644 code/glTF2Asset.h create mode 100644 code/glTF2Asset.inl create mode 100644 code/glTF2AssetWriter.h create mode 100644 code/glTF2AssetWriter.inl create mode 100644 code/glTF2Exporter.cpp create mode 100644 code/glTF2Exporter.h diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h new file mode 100644 index 000000000..41d0dfd06 --- /dev/null +++ b/code/glTF2Asset.h @@ -0,0 +1,1194 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +/** @file glTFAsset.h + * Declares a glTF class to handle gltf/glb files + * + * glTF Extensions Support: + * KHR_binary_glTF: full + * KHR_materials_common: full + */ +#ifndef GLTFASSET_H_INC +#define GLTFASSET_H_INC + +#ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER + +#include +#include +#include +#include +#include +#include + +#define RAPIDJSON_HAS_STDSTRING 1 +#include +#include +#include + +#ifdef ASSIMP_API +# include +# include +# include "ByteSwapper.h" +#else +# include +# define AI_SWAP4(p) +# define ai_assert +#endif + + +#if _MSC_VER > 1500 || (defined __GNUC___) +# define ASSIMP_GLTF_USE_UNORDERED_MULTIMAP +# else +# define gltf_unordered_map map +#endif + +#ifdef ASSIMP_GLTF_USE_UNORDERED_MULTIMAP +# include +# if _MSC_VER > 1600 +# define gltf_unordered_map unordered_map +# else +# define gltf_unordered_map tr1::unordered_map +# endif +#endif + +namespace glTF +{ +#ifdef ASSIMP_API + using Assimp::IOStream; + using Assimp::IOSystem; + using std::shared_ptr; +#else + using std::shared_ptr; + + typedef std::runtime_error DeadlyImportError; + typedef std::runtime_error DeadlyExportError; + + enum aiOrigin { aiOrigin_SET = 0, aiOrigin_CUR = 1, aiOrigin_END = 2 }; + class IOSystem; + class IOStream + { + FILE* f; + public: + IOStream(FILE* file) : f(file) {} + ~IOStream() { fclose(f); f = 0; } + + size_t Read(void* b, size_t sz, size_t n) { return fread(b, sz, n, f); } + size_t Write(const void* b, size_t sz, size_t n) { return fwrite(b, sz, n, f); } + int Seek(size_t off, aiOrigin orig) { return fseek(f, off, int(orig)); } + size_t Tell() const { return ftell(f); } + + size_t FileSize() { + long p = Tell(), len = (Seek(0, aiOrigin_END), Tell()); + return size_t((Seek(p, aiOrigin_SET), len)); + } + }; +#endif + + using rapidjson::Value; + using rapidjson::Document; + + class Asset; + class AssetWriter; + + struct BufferView; // here due to cross-reference + struct Texture; + struct Light; + struct Skin; + + + // Vec/matrix types, as raw float arrays + typedef float (vec3)[3]; + typedef float (vec4)[4]; + typedef float (mat4)[16]; + + + namespace Util + { + void EncodeBase64(const uint8_t* in, size_t inLength, std::string& out); + + size_t DecodeBase64(const char* in, size_t inLength, uint8_t*& out); + + inline size_t DecodeBase64(const char* in, uint8_t*& out) + { + return DecodeBase64(in, strlen(in), out); + } + + struct DataURI + { + const char* mediaType; + const char* charset; + bool base64; + const char* data; + size_t dataLength; + }; + + //! Check if a uri is a data URI + inline bool ParseDataURI(const char* uri, size_t uriLen, DataURI& out); + } + + + //! Magic number for GLB files + #define AI_GLB_MAGIC_NUMBER "glTF" + + #ifdef ASSIMP_API + #include "./../include/assimp/Compiler/pushpack1.h" + #endif + + //! For the KHR_binary_glTF extension (binary .glb file) + //! 20-byte header (+ the JSON + a "body" data section) + struct GLB_Header + { + uint8_t magic[4]; //!< Magic number: "glTF" + uint32_t version; //!< Version number (always 1 as of the last update) + uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes + uint32_t sceneLength; //!< Length, in bytes, of the glTF scene + uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) + } PACK_STRUCT; + + #ifdef ASSIMP_API + #include "./../include/assimp/Compiler/poppack1.h" + #endif + + + //! Values for the GLB_Header::sceneFormat field + enum SceneFormat + { + SceneFormat_JSON = 0 + }; + + //! Values for the mesh primitive modes + enum PrimitiveMode + { + PrimitiveMode_POINTS = 0, + PrimitiveMode_LINES = 1, + PrimitiveMode_LINE_LOOP = 2, + PrimitiveMode_LINE_STRIP = 3, + PrimitiveMode_TRIANGLES = 4, + PrimitiveMode_TRIANGLE_STRIP = 5, + PrimitiveMode_TRIANGLE_FAN = 6 + }; + + //! Values for the Accessor::componentType field + enum ComponentType + { + ComponentType_BYTE = 5120, + ComponentType_UNSIGNED_BYTE = 5121, + ComponentType_SHORT = 5122, + ComponentType_UNSIGNED_SHORT = 5123, + ComponentType_UNSIGNED_INT = 5125, + ComponentType_FLOAT = 5126 + }; + + inline unsigned int ComponentTypeSize(ComponentType t) + { + switch (t) { + case ComponentType_SHORT: + case ComponentType_UNSIGNED_SHORT: + return 2; + + case ComponentType_UNSIGNED_INT: + case ComponentType_FLOAT: + return 4; + + case ComponentType_BYTE: + case ComponentType_UNSIGNED_BYTE: + return 1; + default: + throw DeadlyImportError("GLTF: Unsupported Component Type "+t); + } + } + + //! Values for the BufferView::target field + enum BufferViewTarget + { + BufferViewTarget_ARRAY_BUFFER = 34962, + BufferViewTarget_ELEMENT_ARRAY_BUFFER = 34963 + }; + + //! Values for the Sampler::magFilter field + enum SamplerMagFilter + { + SamplerMagFilter_Nearest = 9728, + SamplerMagFilter_Linear = 9729 + }; + + //! Values for the Sampler::minFilter field + enum SamplerMinFilter + { + SamplerMinFilter_Nearest = 9728, + SamplerMinFilter_Linear = 9729, + SamplerMinFilter_Nearest_Mipmap_Nearest = 9984, + SamplerMinFilter_Linear_Mipmap_Nearest = 9985, + SamplerMinFilter_Nearest_Mipmap_Linear = 9986, + SamplerMinFilter_Linear_Mipmap_Linear = 9987 + }; + + //! Values for the Sampler::wrapS and Sampler::wrapT field + enum SamplerWrap + { + SamplerWrap_Clamp_To_Edge = 33071, + SamplerWrap_Mirrored_Repeat = 33648, + SamplerWrap_Repeat = 10497 + }; + + //! Values for the Texture::format and Texture::internalFormat fields + enum TextureFormat + { + TextureFormat_ALPHA = 6406, + TextureFormat_RGB = 6407, + TextureFormat_RGBA = 6408, + TextureFormat_LUMINANCE = 6409, + TextureFormat_LUMINANCE_ALPHA = 6410 + }; + + //! Values for the Texture::target field + enum TextureTarget + { + TextureTarget_TEXTURE_2D = 3553 + }; + + //! Values for the Texture::type field + enum TextureType + { + TextureType_UNSIGNED_BYTE = 5121, + TextureType_UNSIGNED_SHORT_5_6_5 = 33635, + TextureType_UNSIGNED_SHORT_4_4_4_4 = 32819, + TextureType_UNSIGNED_SHORT_5_5_5_1 = 32820 + }; + + + //! Values for the Accessor::type field (helper class) + class AttribType + { + public: + enum Value + { SCALAR, VEC2, VEC3, VEC4, MAT2, MAT3, MAT4 }; + + private: + static const size_t NUM_VALUES = static_cast(MAT4)+1; + + struct Info + { const char* name; unsigned int numComponents; }; + + template struct data + { static const Info infos[NUM_VALUES]; }; + + public: + inline static Value FromString(const char* str) + { + for (size_t i = 0; i < NUM_VALUES; ++i) { + if (strcmp(data<0>::infos[i].name, str) == 0) { + return static_cast(i); + } + } + return SCALAR; + } + + inline static const char* ToString(Value type) + { + return data<0>::infos[static_cast(type)].name; + } + + inline static unsigned int GetNumComponents(Value type) + { + return data<0>::infos[static_cast(type)].numComponents; + } + }; + + // must match the order of the AttribTypeTraits::Value enum! + template const AttribType::Info + AttribType::data::infos[AttribType::NUM_VALUES] = { + { "SCALAR", 1 }, { "VEC2", 2 }, { "VEC3", 3 }, { "VEC4", 4 }, { "MAT2", 4 }, { "MAT3", 9 }, { "MAT4", 16 } + }; + + + + //! A reference to one top-level object, which is valid + //! until the Asset instance is destroyed + template + class Ref + { + std::vector* vector; + unsigned int index; + + public: + Ref() : vector(0), index(0) {} + Ref(std::vector& vec, unsigned int idx) : vector(&vec), index(idx) {} + + inline unsigned int GetIndex() const + { return index; } + + operator bool() const + { return vector != 0; } + + T* operator->() + { return (*vector)[index]; } + + T& operator*() + { return *((*vector)[index]); } + }; + + //! Helper struct to represent values that might not be present + template + struct Nullable + { + T value; + bool isPresent; + + Nullable() : isPresent(false) {} + Nullable(T& val) : value(val), isPresent(true) {} + }; + + + //! Base classe for all glTF top-level objects + struct Object + { + std::string id; //!< The globally unique ID used to reference this object + std::string name; //!< The user-defined name of this object + + //! Objects marked as special are not exported (used to emulate the binary body buffer) + virtual bool IsSpecial() const + { return false; } + + virtual ~Object() {} + + //! Maps special IDs to another ID, where needed. Subclasses may override it (statically) + static const char* TranslateId(Asset& r, const char* id) + { return id; } + }; + + // + // Classes for each glTF top-level object type + // + + //! A typed view into a BufferView. A BufferView contains raw binary data. + //! An accessor provides a typed view into a BufferView or a subset of a BufferView + //! similar to how WebGL's vertexAttribPointer() defines an attribute in a buffer. + struct Accessor : public Object + { + Ref bufferView; //!< The ID of the bufferView. (required) + unsigned int byteOffset; //!< The offset relative to the start of the bufferView in bytes. (required) + unsigned int byteStride; //!< The stride, in bytes, between attributes referenced by this accessor. (default: 0) + ComponentType componentType; //!< The datatype of components in the attribute. (required) + unsigned int count; //!< The number of attributes referenced by this accessor. (required) + AttribType::Value type; //!< Specifies if the attribute is a scalar, vector, or matrix. (required) + std::vector max; //!< Maximum value of each component in this attribute. + std::vector min; //!< Minimum value of each component in this attribute. + + unsigned int GetNumComponents(); + unsigned int GetBytesPerComponent(); + unsigned int GetElementSize(); + + inline uint8_t* GetPointer(); + + template + bool ExtractData(T*& outData); + + void WriteData(size_t count, const void* src_buffer, size_t src_stride); + + //! Helper class to iterate the data + class Indexer + { + friend struct Accessor; + + Accessor& accessor; + uint8_t* data; + size_t elemSize, stride; + + Indexer(Accessor& acc); + + public: + + //! Accesses the i-th value as defined by the accessor + template + T GetValue(int i); + + //! Accesses the i-th value as defined by the accessor + inline unsigned int GetUInt(int i) + { + return GetValue(i); + } + + inline bool IsValid() const + { + return data != 0; + } + }; + + inline Indexer GetIndexer() + { + return Indexer(*this); + } + + Accessor() {} + void Read(Value& obj, Asset& r); + }; + + //! A buffer points to binary geometry, animation, or skins. + struct Buffer : public Object + { + /********************* Types *********************/ + public: + + enum Type + { + Type_arraybuffer, + Type_text + }; + + /// \struct SEncodedRegion + /// Descriptor of encoded region in "bufferView". + struct SEncodedRegion + { + const size_t Offset;///< Offset from begin of "bufferView" to encoded region, in bytes. + const size_t EncodedData_Length;///< Size of encoded region, in bytes. + uint8_t* const DecodedData;///< Cached encoded data. + const size_t DecodedData_Length;///< Size of decoded region, in bytes. + const std::string ID;///< ID of the region. + + /// \fn SEncodedRegion(const size_t pOffset, const size_t pEncodedData_Length, uint8_t* pDecodedData, const size_t pDecodedData_Length, const std::string pID) + /// Constructor. + /// \param [in] pOffset - offset from begin of "bufferView" to encoded region, in bytes. + /// \param [in] pEncodedData_Length - size of encoded region, in bytes. + /// \param [in] pDecodedData - pointer to decoded data array. + /// \param [in] pDecodedData_Length - size of encoded region, in bytes. + /// \param [in] pID - ID of the region. + SEncodedRegion(const size_t pOffset, const size_t pEncodedData_Length, uint8_t* pDecodedData, const size_t pDecodedData_Length, const std::string pID) + : Offset(pOffset), EncodedData_Length(pEncodedData_Length), DecodedData(pDecodedData), DecodedData_Length(pDecodedData_Length), ID(pID) + {} + + /// \fn ~SEncodedRegion() + /// Destructor. + ~SEncodedRegion() { delete [] DecodedData; } + }; + + /******************* Variables *******************/ + + //std::string uri; //!< The uri of the buffer. Can be a filepath, a data uri, etc. (required) + size_t byteLength; //!< The length of the buffer in bytes. (default: 0) + //std::string type; //!< XMLHttpRequest responseType (default: "arraybuffer") + + Type type; + + /// \var EncodedRegion_Current + /// Pointer to currently active encoded region. + /// Why not decoding all regions at once and not to set one buffer with decoded data? + /// Yes, why not? Even "accessor" point to decoded data. I mean that fields "byteOffset", "byteStride" and "count" has values which describes decoded + /// data array. But only in range of mesh while is active parameters from "compressedData". For another mesh accessors point to decoded data too. But + /// offset is counted for another regions is encoded. + /// Example. You have two meshes. For every of it you have 4 bytes of data. That data compressed to 2 bytes. So, you have buffer with encoded data: + /// M1_E0, M1_E1, M2_E0, M2_E1. + /// After decoding you'll get: + /// M1_D0, M1_D1, M1_D2, M1_D3, M2_D0, M2_D1, M2_D2, M2_D3. + /// "accessors" must to use values that point to decoded data - obviously. So, you'll expect "accessors" like + /// "accessor_0" : { byteOffset: 0, byteLength: 4}, "accessor_1" : { byteOffset: 4, byteLength: 4} + /// but in real life you'll get: + /// "accessor_0" : { byteOffset: 0, byteLength: 4}, "accessor_1" : { byteOffset: 2, byteLength: 4} + /// Yes, accessor of next mesh has offset and length which mean: current mesh data is decoded, all other data is encoded. + /// And when before you start to read data of current mesh (with encoded data ofcourse) you must decode region of "bufferView", after read finished + /// delete encoding mark. And after that you can repeat process: decode data of mesh, read, delete decoded data. + /// + /// Remark. Encoding all data at once is good in world with computers which do not has RAM limitation. So, you must use step by step encoding in + /// exporter and importer. And, thanks to such way, there is no need to load whole file into memory. + SEncodedRegion* EncodedRegion_Current; + + private: + + shared_ptr mData; //!< Pointer to the data + bool mIsSpecial; //!< Set to true for special cases (e.g. the body buffer) + + /// \var EncodedRegion_List + /// List of encoded regions. + std::list EncodedRegion_List; + + /******************* Functions *******************/ + + public: + + Buffer(); + ~Buffer(); + + void Read(Value& obj, Asset& r); + + bool LoadFromStream(IOStream& stream, size_t length = 0, size_t baseOffset = 0); + + /// \fn void EncodedRegion_Mark(const size_t pOffset, const size_t pEncodedData_Length, uint8_t* pDecodedData, const size_t pDecodedData_Length, const std::string& pID) + /// Mark region of "bufferView" as encoded. When data is request from such region then "bufferView" use decoded data. + /// \param [in] pOffset - offset from begin of "bufferView" to encoded region, in bytes. + /// \param [in] pEncodedData_Length - size of encoded region, in bytes. + /// \param [in] pDecodedData - pointer to decoded data array. + /// \param [in] pDecodedData_Length - size of encoded region, in bytes. + /// \param [in] pID - ID of the region. + void EncodedRegion_Mark(const size_t pOffset, const size_t pEncodedData_Length, uint8_t* pDecodedData, const size_t pDecodedData_Length, const std::string& pID); + + /// \fn void EncodedRegion_SetCurrent(const std::string& pID) + /// Select current encoded region by ID. \sa EncodedRegion_Current. + /// \param [in] pID - ID of the region. + void EncodedRegion_SetCurrent(const std::string& pID); + + /// \fn bool ReplaceData(const size_t pBufferData_Offset, const size_t pBufferData_Count, const uint8_t* pReplace_Data, const size_t pReplace_Count) + /// Replace part of buffer data. Pay attention that function work with original array of data (\ref mData) not with encoded regions. + /// \param [in] pBufferData_Offset - index of first element in buffer from which new data will be placed. + /// \param [in] pBufferData_Count - count of bytes in buffer which will be replaced. + /// \param [in] pReplace_Data - pointer to array with new data for buffer. + /// \param [in] pReplace_Count - count of bytes in new data. + /// \return true - if successfully replaced, false if input arguments is out of range. + bool ReplaceData(const size_t pBufferData_Offset, const size_t pBufferData_Count, const uint8_t* pReplace_Data, const size_t pReplace_Count); + + size_t AppendData(uint8_t* data, size_t length); + void Grow(size_t amount); + + uint8_t* GetPointer() + { return mData.get(); } + + void MarkAsSpecial() + { mIsSpecial = true; } + + bool IsSpecial() const + { return mIsSpecial; } + + std::string GetURI() + { return std::string(this->id) + ".bin"; } + + static const char* TranslateId(Asset& r, const char* id); + }; + + //! A view into a buffer generally representing a subset of the buffer. + struct BufferView : public Object + { + Ref buffer; //! The ID of the buffer. (required) + size_t byteOffset; //! The offset into the buffer in bytes. (required) + size_t byteLength; //! The length of the bufferView in bytes. (default: 0) + + BufferViewTarget target; //! The target that the WebGL buffer should be bound to. + + void Read(Value& obj, Asset& r); + }; + + struct Camera : public Object + { + enum Type + { + Perspective, + Orthographic + }; + + Type type; + + union + { + struct { + float aspectRatio; //! bufferView; + + std::string mimeType; + + int width, height; + + private: + uint8_t* mData; + size_t mDataLength; + + public: + + Image(); + void Read(Value& obj, Asset& r); + + inline bool HasData() const + { return mDataLength > 0; } + + inline size_t GetDataLength() const + { return mDataLength; } + + inline const uint8_t* GetData() const + { return mData; } + + inline uint8_t* StealData(); + + inline void SetData(uint8_t* data, size_t length, Asset& r); + }; + + //! Holds a material property that can be a texture or a color + struct TexProperty + { + Ref texture; + vec4 color; + }; + + //! The material appearance of a primitive. + struct Material : public Object + { + //Ref source; //!< The ID of the technique. + //std::gltf_unordered_map values; //!< A dictionary object of parameter values. + + //! Techniques defined by KHR_materials_common + enum Technique + { + Technique_undefined = 0, + Technique_BLINN, + Technique_PHONG, + Technique_LAMBERT, + Technique_CONSTANT + }; + + TexProperty ambient; + TexProperty diffuse; + TexProperty specular; + TexProperty emission; + + bool doubleSided; + bool transparent; + float transparency; + float shininess; + + Technique technique; + + Material() { SetDefaults(); } + void Read(Value& obj, Asset& r); + void SetDefaults(); + }; + + //! A set of primitives to be rendered. A node can contain one or more meshes. A node's transform places the mesh in the scene. + struct Mesh : public Object + { + typedef std::vector< Ref > AccessorList; + + struct Primitive + { + PrimitiveMode mode; + + struct Attributes { + AccessorList position, normal, texcoord, color, joint, jointmatrix, weight; + } attributes; + + Ref indices; + + Ref material; + }; + + /// \struct SExtension + /// Extension used for mesh. + struct SExtension + { + /// \enum EType + /// Type of extension. + enum EType + { + #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + Compression_Open3DGC,///< Compression of mesh data using Open3DGC algorithm. + #endif + + Unknown + }; + + EType Type;///< Type of extension. + + /// \fn SExtension + /// Constructor. + /// \param [in] pType - type of extension. + SExtension(const EType pType) + : Type(pType) + {} + + virtual ~SExtension() { + // empty + } + }; + + #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + /// \struct SCompression_Open3DGC + /// Compression of mesh data using Open3DGC algorithm. + struct SCompression_Open3DGC : public SExtension + { + using SExtension::Type; + + std::string Buffer;///< ID of "buffer" used for storing compressed data. + size_t Offset;///< Offset in "bufferView" where compressed data are stored. + size_t Count;///< Count of elements in compressed data. Is always equivalent to size in bytes: look comments for "Type" and "Component_Type". + bool Binary;///< If true then "binary" mode is used for coding, if false - "ascii" mode. + size_t IndicesCount;///< Count of indices in mesh. + size_t VerticesCount;///< Count of vertices in mesh. + // AttribType::Value Type;///< Is always "SCALAR". + // ComponentType Component_Type;///< Is always "ComponentType_UNSIGNED_BYTE" (5121). + + /// \fn SCompression_Open3DGC + /// Constructor. + SCompression_Open3DGC() + : SExtension(Compression_Open3DGC) { + // empty + } + + virtual ~SCompression_Open3DGC() { + // empty + } + }; + #endif + + std::vector primitives; + std::list Extension;///< List of extensions used in mesh. + + Mesh() {} + + /// \fn ~Mesh() + /// Destructor. + ~Mesh() { for(std::list::iterator it = Extension.begin(), it_end = Extension.end(); it != it_end; it++) { delete *it; }; } + + /// \fn void Read(Value& pJSON_Object, Asset& pAsset_Root) + /// Get mesh data from JSON-object and place them to root asset. + /// \param [in] pJSON_Object - reference to pJSON-object from which data are read. + /// \param [out] pAsset_Root - reference to root assed where data will be stored. + void Read(Value& pJSON_Object, Asset& pAsset_Root); + + #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + /// \fn void Decode_O3DGC(const SCompression_Open3DGC& pCompression_Open3DGC, Asset& pAsset_Root) + /// Decode part of "buffer" which encoded with Open3DGC algorithm. + /// \param [in] pCompression_Open3DGC - reference to structure which describe encoded region. + /// \param [out] pAsset_Root - reference to root assed where data will be stored. + void Decode_O3DGC(const SCompression_Open3DGC& pCompression_Open3DGC, Asset& pAsset_Root); + #endif + }; + + struct Node : public Object + { + std::vector< Ref > children; + std::vector< Ref > meshes; + + Nullable matrix; + Nullable translation; + Nullable rotation; + Nullable scale; + + Ref camera; + Ref light; + + std::vector< Ref > skeletons; //!< The ID of skeleton nodes. Each of which is the root of a node hierarchy. + Ref skin; //!< The ID of the skin referenced by this node. + std::string jointName; //!< Name used when this node is a joint in a skin. + + Ref parent; //!< This is not part of the glTF specification. Used as a helper. + + Node() {} + void Read(Value& obj, Asset& r); + }; + + struct Program : public Object + { + Program() {} + void Read(Value& obj, Asset& r); + }; + + + struct Sampler : public Object + { + SamplerMagFilter magFilter; //!< The texture magnification filter. (required) + SamplerMinFilter minFilter; //!< The texture minification filter. (required) + SamplerWrap wrapS; //!< The texture wrapping in the S direction. (required) + SamplerWrap wrapT; //!< The texture wrapping in the T direction. (required) + + Sampler() {} + void Read(Value& obj, Asset& r); + void SetDefaults(); + }; + + struct Scene : public Object + { + std::vector< Ref > nodes; + + Scene() {} + void Read(Value& obj, Asset& r); + }; + + struct Shader : public Object + { + Shader() {} + void Read(Value& obj, Asset& r); + }; + + struct Skin : public Object + { + Nullable bindShapeMatrix; //!< Floating-point 4x4 transformation matrix stored in column-major order. + Ref inverseBindMatrices; //!< The ID of the accessor containing the floating-point 4x4 inverse-bind matrices. + std::vector> jointNames; //!< Joint names of the joints (nodes with a jointName property) in this skin. + std::string name; //!< The user-defined name of this object. + + Skin() {} + void Read(Value& obj, Asset& r); + }; + + struct Technique : public Object + { + struct Parameters + { + + }; + + struct States + { + + }; + + struct Functions + { + + }; + + Technique() {} + void Read(Value& obj, Asset& r); + }; + + //! A texture and its sampler. + struct Texture : public Object + { + Ref sampler; //!< The ID of the sampler used by this texture. (required) + Ref source; //!< The ID of the image used by this texture. (required) + + //TextureFormat format; //!< The texture's format. (default: TextureFormat_RGBA) + //TextureFormat internalFormat; //!< The texture's internal format. (default: TextureFormat_RGBA) + + //TextureTarget target; //!< The target that the WebGL texture should be bound to. (default: TextureTarget_TEXTURE_2D) + //TextureType type; //!< Texel datatype. (default: TextureType_UNSIGNED_BYTE) + + Texture() {} + void Read(Value& obj, Asset& r); + }; + + + //! A light (from KHR_materials_common extension) + struct Light : public Object + { + enum Type + { + Type_undefined, + Type_ambient, + Type_directional, + Type_point, + Type_spot + }; + + Type type; + + vec4 color; + float distance; + float constantAttenuation; + float linearAttenuation; + float quadraticAttenuation; + float falloffAngle; + float falloffExponent; + + Light() {} + void Read(Value& obj, Asset& r); + + void SetDefaults(); + }; + + struct Animation : public Object + { + struct AnimSampler { + std::string id; //!< The ID of this sampler. + std::string input; //!< The ID of a parameter in this animation to use as key-frame input. + std::string interpolation; //!< Type of interpolation algorithm to use between key-frames. + std::string output; //!< The ID of a parameter in this animation to use as key-frame output. + }; + + struct AnimChannel { + std::string sampler; //!< The ID of one sampler present in the containing animation's samplers property. + + struct AnimTarget { + Ref id; //!< The ID of the node to animate. + std::string path; //!< The name of property of the node to animate ("translation", "rotation", or "scale"). + } target; + }; + + struct AnimParameters { + Ref TIME; //!< Accessor reference to a buffer storing a array of floating point scalar values. + Ref rotation; //!< Accessor reference to a buffer storing a array of four-component floating-point vectors. + Ref scale; //!< Accessor reference to a buffer storing a array of three-component floating-point vectors. + Ref translation; //!< Accessor reference to a buffer storing a array of three-component floating-point vectors. + }; + + // AnimChannel Channels[3]; //!< Connect the output values of the key-frame animation to a specific node in the hierarchy. + // AnimParameters Parameters; //!< The samplers that interpolate between the key-frames. + // AnimSampler Samplers[3]; //!< The parameterized inputs representing the key-frame data. + + std::vector Channels; //!< Connect the output values of the key-frame animation to a specific node in the hierarchy. + AnimParameters Parameters; //!< The samplers that interpolate between the key-frames. + std::vector Samplers; //!< The parameterized inputs representing the key-frame data. + + Animation() {} + void Read(Value& obj, Asset& r); + }; + + + //! Base class for LazyDict that acts as an interface + class LazyDictBase + { + public: + virtual ~LazyDictBase() {} + + virtual void AttachToDocument(Document& doc) = 0; + virtual void DetachFromDocument() = 0; + + virtual void WriteObjects(AssetWriter& writer) = 0; + }; + + + template + class LazyDict; + + //! (Implemented in glTFAssetWriter.h) + template + void WriteLazyDict(LazyDict& d, AssetWriter& w); + + + //! Manages lazy loading of the glTF top-level objects, and keeps a reference to them by ID + //! It is the owner the loaded objects, so when it is destroyed it also deletes them + template + class LazyDict : public LazyDictBase + { + friend class Asset; + friend class AssetWriter; + + typedef typename std::gltf_unordered_map< std::string, unsigned int > Dict; + + std::vector mObjs; //! The read objects + Dict mObjsById; //! The read objects accessible by id + const char* mDictId; //! ID of the dictionary object + const char* mExtId; //! ID of the extension defining the dictionary + Value* mDict; //! JSON dictionary object + Asset& mAsset; //! The asset instance + + void AttachToDocument(Document& doc); + void DetachFromDocument(); + + void WriteObjects(AssetWriter& writer) + { WriteLazyDict(*this, writer); } + + Ref Add(T* obj); + + public: + LazyDict(Asset& asset, const char* dictId, const char* extId = 0); + ~LazyDict(); + + Ref Get(const char* id); + Ref Get(unsigned int i); + Ref Get(const std::string& pID) { return Get(pID.c_str()); } + + Ref Create(const char* id); + Ref Create(const std::string& id) + { return Create(id.c_str()); } + + inline unsigned int Size() const + { return unsigned(mObjs.size()); } + + inline T& operator[](size_t i) + { return *mObjs[i]; } + + }; + + + struct AssetMetadata + { + std::string copyright; //!< A copyright message suitable for display to credit the content creator. + std::string generator; //!< Tool that generated this glTF model.Useful for debugging. + bool premultipliedAlpha; //!< Specifies if the shaders were generated with premultiplied alpha. (default: false) + + struct { + std::string api; //!< Specifies the target rendering API (default: "WebGL") + std::string version; //!< Specifies the target rendering API (default: "1.0.3") + } profile; //!< Specifies the target rendering API and version, e.g., WebGL 1.0.3. (default: {}) + + int version; //!< The glTF format version (should be 1) + + void Read(Document& doc); + + AssetMetadata() + : premultipliedAlpha(false) + , version(0) + { + } + }; + + // + // glTF Asset class + // + + //! Root object for a glTF asset + class Asset + { + typedef std::gltf_unordered_map IdMap; + + template + friend class LazyDict; + + friend struct Buffer; // To access OpenFile + + friend class AssetWriter; + + private: + IOSystem* mIOSystem; + + std::string mCurrentAssetDir; + + size_t mSceneLength; + size_t mBodyOffset, mBodyLength; + + std::vector mDicts; + + IdMap mUsedIds; + + Ref mBodyBuffer; + + Asset(Asset&); + Asset& operator=(const Asset&); + + public: + + //! Keeps info about the enabled extensions + struct Extensions + { + bool KHR_binary_glTF; + bool KHR_materials_common; + + } extensionsUsed; + + AssetMetadata asset; + + + // Dictionaries for each type of object + + LazyDict accessors; + LazyDict animations; + LazyDict buffers; + LazyDict bufferViews; + LazyDict cameras; + LazyDict images; + LazyDict materials; + LazyDict meshes; + LazyDict nodes; + //LazyDict programs; + LazyDict samplers; + LazyDict scenes; + //LazyDict shaders; + LazyDict skins; + //LazyDict techniques; + LazyDict textures; + + LazyDict lights; // KHR_materials_common ext + + Ref scene; + + public: + Asset(IOSystem* io = 0) + : mIOSystem(io) + , asset() + , accessors (*this, "accessors") + , animations (*this, "animations") + , buffers (*this, "buffers") + , bufferViews (*this, "bufferViews") + , cameras (*this, "cameras") + , images (*this, "images") + , materials (*this, "materials") + , meshes (*this, "meshes") + , nodes (*this, "nodes") + //, programs (*this, "programs") + , samplers (*this, "samplers") + , scenes (*this, "scenes") + //, shaders (*this, "shaders") + , skins (*this, "skins") + //, techniques (*this, "techniques") + , textures (*this, "textures") + , lights (*this, "lights", "KHR_materials_common") + { + memset(&extensionsUsed, 0, sizeof(extensionsUsed)); + } + + //! Main function + void Load(const std::string& file, bool isBinary = false); + + //! Enables the "KHR_binary_glTF" extension on the asset + void SetAsBinary(); + + //! Search for an available name, starting from the given strings + std::string FindUniqueID(const std::string& str, const char* suffix); + + Ref GetBodyBuffer() + { return mBodyBuffer; } + + private: + void ReadBinaryHeader(IOStream& stream); + + void ReadExtensionsUsed(Document& doc); + + + IOStream* OpenFile(std::string path, const char* mode, bool absolute = false); + }; + +} + +// Include the implementation of the methods +#include "glTFAsset.inl" + +#endif // ASSIMP_BUILD_NO_GLTF_IMPORTER + +#endif // GLTFASSET_H_INC diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl new file mode 100644 index 000000000..fe29dde70 --- /dev/null +++ b/code/glTF2Asset.inl @@ -0,0 +1,1636 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +#include "StringUtils.h" + +// Header files, Assimp +#include + +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + // Header files, Open3DGC. +# include +#endif + +using namespace Assimp; + +namespace glTF { + +namespace { + + // + // JSON Value reading helpers + // + + template + struct ReadHelper { static bool Read(Value& val, T& out) { + return val.IsInt() ? out = static_cast(val.GetInt()), true : false; + }}; + + template<> struct ReadHelper { static bool Read(Value& val, bool& out) { + return val.IsBool() ? out = val.GetBool(), true : false; + }}; + + template<> struct ReadHelper { static bool Read(Value& val, float& out) { + return val.IsNumber() ? out = static_cast(val.GetDouble()), true : false; + }}; + + template struct ReadHelper { static bool Read(Value& val, float (&out)[N]) { + if (!val.IsArray() || val.Size() != N) return false; + for (unsigned int i = 0; i < N; ++i) { + if (val[i].IsNumber()) + out[i] = static_cast(val[i].GetDouble()); + } + return true; + }}; + + template<> struct ReadHelper { static bool Read(Value& val, const char*& out) { + return val.IsString() ? (out = val.GetString(), true) : false; + }}; + + template<> struct ReadHelper { static bool Read(Value& val, std::string& out) { + return val.IsString() ? (out = std::string(val.GetString(), val.GetStringLength()), true) : false; + }}; + + template struct ReadHelper< Nullable > { static bool Read(Value& val, Nullable& out) { + return out.isPresent = ReadHelper::Read(val, out.value); + }}; + + template + inline static bool ReadValue(Value& val, T& out) + { + return ReadHelper::Read(val, out); + } + + template + inline static bool ReadMember(Value& obj, const char* id, T& out) + { + Value::MemberIterator it = obj.FindMember(id); + if (it != obj.MemberEnd()) { + return ReadHelper::Read(it->value, out); + } + return false; + } + + template + inline static T MemberOrDefault(Value& obj, const char* id, T defaultValue) + { + T out; + return ReadMember(obj, id, out) ? out : defaultValue; + } + + inline Value* FindMember(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd()) ? &it->value : 0; + } + + inline Value* FindString(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd() && it->value.IsString()) ? &it->value : 0; + } + + inline Value* FindArray(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd() && it->value.IsArray()) ? &it->value : 0; + } + + inline Value* FindObject(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd() && it->value.IsObject()) ? &it->value : 0; + } +} + +// +// LazyDict methods +// + +template +inline LazyDict::LazyDict(Asset& asset, const char* dictId, const char* extId) + : mDictId(dictId), mExtId(extId), mDict(0), mAsset(asset) +{ + asset.mDicts.push_back(this); // register to the list of dictionaries +} + +template +inline LazyDict::~LazyDict() +{ + for (size_t i = 0; i < mObjs.size(); ++i) { + delete mObjs[i]; + } +} + + +template +inline void LazyDict::AttachToDocument(Document& doc) +{ + Value* container = 0; + + if (mExtId) { + if (Value* exts = FindObject(doc, "extensions")) { + container = FindObject(*exts, mExtId); + } + } + else { + container = &doc; + } + + if (container) { + mDict = FindObject(*container, mDictId); + } +} + +template +inline void LazyDict::DetachFromDocument() +{ + mDict = 0; +} + +template +Ref LazyDict::Get(unsigned int i) +{ + return Ref(mObjs, i); +} + +template +Ref LazyDict::Get(const char* id) +{ + id = T::TranslateId(mAsset, id); + + typename Dict::iterator it = mObjsById.find(id); + if (it != mObjsById.end()) { // already created? + return Ref(mObjs, it->second); + } + + // read it from the JSON object + if (!mDict) { + throw DeadlyImportError("GLTF: Missing section \"" + std::string(mDictId) + "\""); + } + + Value::MemberIterator obj = mDict->FindMember(id); + if (obj == mDict->MemberEnd()) { + throw DeadlyImportError("GLTF: Missing object with id \"" + std::string(id) + "\" in \"" + mDictId + "\""); + } + if (!obj->value.IsObject()) { + throw DeadlyImportError("GLTF: Object with id \"" + std::string(id) + "\" is not a JSON object"); + } + + // create an instance of the given type + T* inst = new T(); + inst->id = id; + ReadMember(obj->value, "name", inst->name); + inst->Read(obj->value, mAsset); + return Add(inst); +} + +template +Ref LazyDict::Add(T* obj) +{ + unsigned int idx = unsigned(mObjs.size()); + mObjs.push_back(obj); + mObjsById[obj->id] = idx; + mAsset.mUsedIds[obj->id] = true; + return Ref(mObjs, idx); +} + +template +Ref LazyDict::Create(const char* id) +{ + Asset::IdMap::iterator it = mAsset.mUsedIds.find(id); + if (it != mAsset.mUsedIds.end()) { + throw DeadlyImportError("GLTF: two objects with the same ID exist"); + } + T* inst = new T(); + inst->id = id; + return Add(inst); +} + + +// +// glTF dictionary objects methods +// + + +inline Buffer::Buffer() + : byteLength(0), type(Type_arraybuffer), EncodedRegion_Current(nullptr), mIsSpecial(false) +{ } + +inline Buffer::~Buffer() +{ + for(SEncodedRegion* reg : EncodedRegion_List) delete reg; +} + +inline const char* Buffer::TranslateId(Asset& r, const char* id) +{ + // Compatibility with old spec + if (r.extensionsUsed.KHR_binary_glTF && strcmp(id, "KHR_binary_glTF") == 0) { + return "binary_glTF"; + } + + return id; +} + +inline void Buffer::Read(Value& obj, Asset& r) +{ + size_t statedLength = MemberOrDefault(obj, "byteLength", 0); + byteLength = statedLength; + + Value* it = FindString(obj, "uri"); + if (!it) { + if (statedLength > 0) { + throw DeadlyImportError("GLTF: buffer with non-zero length missing the \"uri\" attribute"); + } + return; + } + + const char* uri = it->GetString(); + + Util::DataURI dataURI; + if (ParseDataURI(uri, it->GetStringLength(), dataURI)) { + if (dataURI.base64) { + uint8_t* data = 0; + this->byteLength = Util::DecodeBase64(dataURI.data, dataURI.dataLength, data); + this->mData.reset(data); + + if (statedLength > 0 && this->byteLength != statedLength) { + throw DeadlyImportError("GLTF: buffer \"" + id + "\", expected " + to_string(statedLength) + + " bytes, but found " + to_string(dataURI.dataLength)); + } + } + else { // assume raw data + if (statedLength != dataURI.dataLength) { + throw DeadlyImportError("GLTF: buffer \"" + id + "\", expected " + to_string(statedLength) + + " bytes, but found " + to_string(dataURI.dataLength)); + } + + this->mData.reset(new uint8_t[dataURI.dataLength]); + memcpy( this->mData.get(), dataURI.data, dataURI.dataLength ); + } + } + else { // Local file + if (byteLength > 0) { + IOStream* file = r.OpenFile(uri, "rb"); + if (file) { + bool ok = LoadFromStream(*file, byteLength); + delete file; + + if (!ok) + throw DeadlyImportError("GLTF: error while reading referenced file \"" + std::string(uri) + "\"" ); + } + else { + throw DeadlyImportError("GLTF: could not open referenced file \"" + std::string(uri) + "\""); + } + } + } +} + +inline bool Buffer::LoadFromStream(IOStream& stream, size_t length, size_t baseOffset) +{ + byteLength = length ? length : stream.FileSize(); + + if (baseOffset) { + stream.Seek(baseOffset, aiOrigin_SET); + } + + mData.reset(new uint8_t[byteLength]); + + if (stream.Read(mData.get(), byteLength, 1) != 1) { + return false; + } + return true; +} + +inline void Buffer::EncodedRegion_Mark(const size_t pOffset, const size_t pEncodedData_Length, uint8_t* pDecodedData, const size_t pDecodedData_Length, const std::string& pID) +{ + // Check pointer to data + if(pDecodedData == nullptr) throw DeadlyImportError("GLTF: for marking encoded region pointer to decoded data must be provided."); + + // Check offset + if(pOffset > byteLength) + { + const uint8_t val_size = 32; + + char val[val_size]; + + ai_snprintf(val, val_size, "%llu", (long long)pOffset); + throw DeadlyImportError(std::string("GLTF: incorrect offset value (") + val + ") for marking encoded region."); + } + + // Check length + if((pOffset + pEncodedData_Length) > byteLength) + { + const uint8_t val_size = 64; + + char val[val_size]; + + ai_snprintf(val, val_size, "%llu, %llu", (long long)pOffset, (long long)pEncodedData_Length); + throw DeadlyImportError(std::string("GLTF: encoded region with offset/length (") + val + ") is out of range."); + } + + // Add new region + EncodedRegion_List.push_back(new SEncodedRegion(pOffset, pEncodedData_Length, pDecodedData, pDecodedData_Length, pID)); + // And set new value for "byteLength" + byteLength += (pDecodedData_Length - pEncodedData_Length); +} + +inline void Buffer::EncodedRegion_SetCurrent(const std::string& pID) +{ + if((EncodedRegion_Current != nullptr) && (EncodedRegion_Current->ID == pID)) return; + + for(SEncodedRegion* reg : EncodedRegion_List) + { + if(reg->ID == pID) + { + EncodedRegion_Current = reg; + + return; + } + + } + + throw DeadlyImportError("GLTF: EncodedRegion with ID: \"" + pID + "\" not found."); +} + +inline bool Buffer::ReplaceData(const size_t pBufferData_Offset, const size_t pBufferData_Count, const uint8_t* pReplace_Data, const size_t pReplace_Count) +{ +const size_t new_data_size = byteLength + pReplace_Count - pBufferData_Count; + +uint8_t* new_data; + + if((pBufferData_Count == 0) || (pReplace_Count == 0) || (pReplace_Data == nullptr)) return false; + + new_data = new uint8_t[new_data_size]; + // Copy data which place before replacing part. + memcpy(new_data, mData.get(), pBufferData_Offset); + // Copy new data. + memcpy(&new_data[pBufferData_Offset], pReplace_Data, pReplace_Count); + // Copy data which place after replacing part. + memcpy(&new_data[pBufferData_Offset + pReplace_Count], &mData.get()[pBufferData_Offset + pBufferData_Count], pBufferData_Offset); + // Apply new data + mData.reset(new_data); + byteLength = new_data_size; + + return true; +} + +inline size_t Buffer::AppendData(uint8_t* data, size_t length) +{ + size_t offset = this->byteLength; + Grow(length); + memcpy(mData.get() + offset, data, length); + return offset; +} + +inline void Buffer::Grow(size_t amount) +{ + if (amount <= 0) return; + uint8_t* b = new uint8_t[byteLength + amount]; + if (mData) memcpy(b, mData.get(), byteLength); + mData.reset(b); + byteLength += amount; +} + +// +// struct BufferView +// + +inline void BufferView::Read(Value& obj, Asset& r) +{ + const char* bufferId = MemberOrDefault(obj, "buffer", 0); + if (bufferId) { + buffer = r.buffers.Get(bufferId); + } + + byteOffset = MemberOrDefault(obj, "byteOffset", 0u); + byteLength = MemberOrDefault(obj, "byteLength", 0u); +} + +// +// struct Accessor +// + +inline void Accessor::Read(Value& obj, Asset& r) +{ + const char* bufferViewId = MemberOrDefault(obj, "bufferView", 0); + if (bufferViewId) { + bufferView = r.bufferViews.Get(bufferViewId); + } + + byteOffset = MemberOrDefault(obj, "byteOffset", 0u); + byteStride = MemberOrDefault(obj, "byteStride", 0u); + componentType = MemberOrDefault(obj, "componentType", ComponentType_BYTE); + count = MemberOrDefault(obj, "count", 0u); + + const char* typestr; + type = ReadMember(obj, "type", typestr) ? AttribType::FromString(typestr) : AttribType::SCALAR; +} + +inline unsigned int Accessor::GetNumComponents() +{ + return AttribType::GetNumComponents(type); +} + +inline unsigned int Accessor::GetBytesPerComponent() +{ + return int(ComponentTypeSize(componentType)); +} + +inline unsigned int Accessor::GetElementSize() +{ + return GetNumComponents() * GetBytesPerComponent(); +} + +inline uint8_t* Accessor::GetPointer() +{ + if (!bufferView || !bufferView->buffer) return 0; + uint8_t* basePtr = bufferView->buffer->GetPointer(); + if (!basePtr) return 0; + + size_t offset = byteOffset + bufferView->byteOffset; + + // Check if region is encoded. + if(bufferView->buffer->EncodedRegion_Current != nullptr) + { + const size_t begin = bufferView->buffer->EncodedRegion_Current->Offset; + const size_t end = begin + bufferView->buffer->EncodedRegion_Current->DecodedData_Length; + + if((offset >= begin) && (offset < end)) + return &bufferView->buffer->EncodedRegion_Current->DecodedData[offset - begin]; + } + + return basePtr + offset; +} + +namespace { + inline void CopyData(size_t count, + const uint8_t* src, size_t src_stride, + uint8_t* dst, size_t dst_stride) + { + if (src_stride == dst_stride) { + memcpy(dst, src, count * src_stride); + } + else { + size_t sz = std::min(src_stride, dst_stride); + for (size_t i = 0; i < count; ++i) { + memcpy(dst, src, sz); + if (sz < dst_stride) { + memset(dst + sz, 0, dst_stride - sz); + } + src += src_stride; + dst += dst_stride; + } + } + } +} + +template +bool Accessor::ExtractData(T*& outData) +{ + uint8_t* data = GetPointer(); + if (!data) return false; + + const size_t elemSize = GetElementSize(); + const size_t totalSize = elemSize * count; + + const size_t stride = byteStride ? byteStride : elemSize; + + const size_t targetElemSize = sizeof(T); + ai_assert(elemSize <= targetElemSize); + + ai_assert(count*stride <= bufferView->byteLength); + + outData = new T[count]; + if (stride == elemSize && targetElemSize == elemSize) { + memcpy(outData, data, totalSize); + } + else { + for (size_t i = 0; i < count; ++i) { + memcpy(outData + i, data + i*stride, elemSize); + } + } + + return true; +} + +inline void Accessor::WriteData(size_t count, const void* src_buffer, size_t src_stride) +{ + uint8_t* buffer_ptr = bufferView->buffer->GetPointer(); + size_t offset = byteOffset + bufferView->byteOffset; + + size_t dst_stride = GetNumComponents() * GetBytesPerComponent(); + + const uint8_t* src = reinterpret_cast(src_buffer); + uint8_t* dst = reinterpret_cast< uint8_t*>(buffer_ptr + offset); + + ai_assert(dst + count*dst_stride <= buffer_ptr + bufferView->buffer->byteLength); + CopyData(count, src, src_stride, dst, dst_stride); +} + + + +inline Accessor::Indexer::Indexer(Accessor& acc) + : accessor(acc) + , data(acc.GetPointer()) + , elemSize(acc.GetElementSize()) + , stride(acc.byteStride ? acc.byteStride : elemSize) +{ + +} + +//! Accesses the i-th value as defined by the accessor +template +T Accessor::Indexer::GetValue(int i) +{ + ai_assert(data); + ai_assert(i*stride < accessor.bufferView->byteLength); + T value = T(); + memcpy(&value, data + i*stride, elemSize); + //value >>= 8 * (sizeof(T) - elemSize); + return value; +} + +inline Image::Image() + : width(0) + , height(0) + , mData(0) + , mDataLength(0) +{ + +} + +inline void Image::Read(Value& obj, Asset& r) +{ + // Check for extensions first (to detect binary embedded data) + if (Value* extensions = FindObject(obj, "extensions")) { + if (r.extensionsUsed.KHR_binary_glTF) { + if (Value* ext = FindObject(*extensions, "KHR_binary_glTF")) { + + width = MemberOrDefault(*ext, "width", 0); + height = MemberOrDefault(*ext, "height", 0); + + ReadMember(*ext, "mimeType", mimeType); + + const char* bufferViewId; + if (ReadMember(*ext, "bufferView", bufferViewId)) { + Ref bv = r.bufferViews.Get(bufferViewId); + if (bv) { + mDataLength = bv->byteLength; + mData = new uint8_t[mDataLength]; + memcpy(mData, bv->buffer->GetPointer() + bv->byteOffset, mDataLength); + } + } + } + } + } + + if (!mDataLength) { + if (Value* uri = FindString(obj, "uri")) { + const char* uristr = uri->GetString(); + + Util::DataURI dataURI; + if (ParseDataURI(uristr, uri->GetStringLength(), dataURI)) { + mimeType = dataURI.mediaType; + if (dataURI.base64) { + mDataLength = Util::DecodeBase64(dataURI.data, dataURI.dataLength, mData); + } + } + else { + this->uri = uristr; + } + } + } +} + +inline uint8_t* Image::StealData() +{ + uint8_t* data = mData; + mDataLength = 0; + mData = 0; + return data; +} + +inline void Image::SetData(uint8_t* data, size_t length, Asset& r) +{ + Ref b = r.GetBodyBuffer(); + if (b) { // binary file: append to body + std::string bvId = r.FindUniqueID(this->id, "imgdata"); + bufferView = r.bufferViews.Create(bvId); + + bufferView->buffer = b; + bufferView->byteLength = length; + bufferView->byteOffset = b->AppendData(data, length); + } + else { // text file: will be stored as a data uri + this->mData = data; + this->mDataLength = length; + } +} + +inline void Sampler::Read(Value& obj, Asset& r) +{ + SetDefaults(); + + ReadMember(obj, "magFilter", magFilter); + ReadMember(obj, "minFilter", minFilter); + ReadMember(obj, "wrapS", wrapS); + ReadMember(obj, "wrapT", wrapT); +} + +inline void Sampler::SetDefaults() +{ + magFilter = SamplerMagFilter_Linear; + minFilter = SamplerMinFilter_Linear; + wrapS = SamplerWrap_Repeat; + wrapT = SamplerWrap_Repeat; +} + +inline void Texture::Read(Value& obj, Asset& r) +{ + const char* sourcestr; + if (ReadMember(obj, "source", sourcestr)) { + source = r.images.Get(sourcestr); + } + + const char* samplerstr; + if (ReadMember(obj, "sampler", samplerstr)) { + sampler = r.samplers.Get(samplerstr); + } +} + +namespace { + inline void ReadMaterialProperty(Asset& r, Value& vals, const char* propName, TexProperty& out) + { + if (Value* prop = FindMember(vals, propName)) { + if (prop->IsString()) { + out.texture = r.textures.Get(prop->GetString()); + } + else { + ReadValue(*prop, out.color); + } + } + } +} + +inline void Material::Read(Value& material, Asset& r) +{ + SetDefaults(); + + if (Value* values = FindObject(material, "values")) { + ReadMaterialProperty(r, *values, "ambient", this->ambient); + ReadMaterialProperty(r, *values, "diffuse", this->diffuse); + ReadMaterialProperty(r, *values, "specular", this->specular); + + ReadMember(*values, "transparency", transparency); + ReadMember(*values, "shininess", shininess); + } + + if (Value* extensions = FindObject(material, "extensions")) { + if (r.extensionsUsed.KHR_materials_common) { + if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { + if (Value* tnq = FindString(*ext, "technique")) { + const char* t = tnq->GetString(); + if (strcmp(t, "BLINN") == 0) technique = Technique_BLINN; + else if (strcmp(t, "PHONG") == 0) technique = Technique_PHONG; + else if (strcmp(t, "LAMBERT") == 0) technique = Technique_LAMBERT; + else if (strcmp(t, "CONSTANT") == 0) technique = Technique_CONSTANT; + } + + if (Value* values = FindObject(*ext, "values")) { + ReadMaterialProperty(r, *values, "ambient", this->ambient); + ReadMaterialProperty(r, *values, "diffuse", this->diffuse); + ReadMaterialProperty(r, *values, "specular", this->specular); + + ReadMember(*values, "doubleSided", doubleSided); + ReadMember(*values, "transparent", transparent); + ReadMember(*values, "transparency", transparency); + ReadMember(*values, "shininess", shininess); + } + } + } + } +} + +namespace { + void SetVector(vec4& v, float x, float y, float z, float w) + { v[0] = x; v[1] = y; v[2] = z; v[3] = w; } +} + +inline void Material::SetDefaults() +{ + SetVector(ambient.color, 0, 0, 0, 1); + SetVector(diffuse.color, 0, 0, 0, 1); + SetVector(specular.color, 0, 0, 0, 1); + SetVector(emission.color, 0, 0, 0, 1); + + doubleSided = false; + transparent = false; + transparency = 1.0; + shininess = 0.0; + + technique = Technique_undefined; +} + +namespace { + + template + inline int Compare(const char* attr, const char (&str)[N]) { + return (strncmp(attr, str, N - 1) == 0) ? N - 1 : 0; + } + + inline bool GetAttribVector(Mesh::Primitive& p, const char* attr, Mesh::AccessorList*& v, int& pos) + { + if ((pos = Compare(attr, "POSITION"))) { + v = &(p.attributes.position); + } + else if ((pos = Compare(attr, "NORMAL"))) { + v = &(p.attributes.normal); + } + else if ((pos = Compare(attr, "TEXCOORD"))) { + v = &(p.attributes.texcoord); + } + else if ((pos = Compare(attr, "COLOR"))) { + v = &(p.attributes.color); + } + else if ((pos = Compare(attr, "JOINT"))) { + v = &(p.attributes.joint); + } + else if ((pos = Compare(attr, "JOINTMATRIX"))) { + v = &(p.attributes.jointmatrix); + } + else if ((pos = Compare(attr, "WEIGHT"))) { + v = &(p.attributes.weight); + } + else return false; + return true; + } +} + +inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) +{ + /****************** Mesh primitives ******************/ + if (Value* primitives = FindArray(pJSON_Object, "primitives")) { + this->primitives.resize(primitives->Size()); + for (unsigned int i = 0; i < primitives->Size(); ++i) { + Value& primitive = (*primitives)[i]; + + Primitive& prim = this->primitives[i]; + prim.mode = MemberOrDefault(primitive, "mode", PrimitiveMode_TRIANGLES); + + if (Value* attrs = FindObject(primitive, "attributes")) { + for (Value::MemberIterator it = attrs->MemberBegin(); it != attrs->MemberEnd(); ++it) { + if (!it->value.IsString()) continue; + const char* attr = it->name.GetString(); + // Valid attribute semantics include POSITION, NORMAL, TEXCOORD, COLOR, JOINT, JOINTMATRIX, + // and WEIGHT.Attribute semantics can be of the form[semantic]_[set_index], e.g., TEXCOORD_0, TEXCOORD_1, etc. + + int undPos = 0; + Mesh::AccessorList* vec = 0; + if (GetAttribVector(prim, attr, vec, undPos)) { + size_t idx = (attr[undPos] == '_') ? atoi(attr + undPos + 1) : 0; + if ((*vec).size() <= idx) (*vec).resize(idx + 1); + (*vec)[idx] = pAsset_Root.accessors.Get(it->value.GetString()); + } + } + } + + if (Value* indices = FindString(primitive, "indices")) { + prim.indices = pAsset_Root.accessors.Get(indices->GetString()); + } + + if (Value* material = FindString(primitive, "material")) { + prim.material = pAsset_Root.materials.Get(material->GetString()); + } + } + } + + /****************** Mesh extensions ******************/ + Value* json_extensions = FindObject(pJSON_Object, "extensions"); + + if(json_extensions == nullptr) goto mr_skip_extensions; + + for(Value::MemberIterator it_memb = json_extensions->MemberBegin(); it_memb != json_extensions->MemberEnd(); it_memb++) + { +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + if(it_memb->name.GetString() == std::string("Open3DGC-compression")) + { + // Search for compressed data. + // Compressed data contain description of part of "buffer" which is encoded. This part must be decoded and + // new data will replace old encoded part by request. In fact \"compressedData\" is kind of "accessor" structure. + Value* comp_data = FindObject(it_memb->value, "compressedData"); + + if(comp_data == nullptr) throw DeadlyImportError("GLTF: \"Open3DGC-compression\" must has \"compressedData\"."); + + DefaultLogger::get()->info("GLTF: Decompressing Open3DGC data."); + + /************** Read data from JSON-document **************/ + #define MESH_READ_COMPRESSEDDATA_MEMBER(pFieldName, pOut) \ + if(!ReadMember(*comp_data, pFieldName, pOut)) \ + { \ + throw DeadlyImportError(std::string("GLTF: \"compressedData\" must has \"") + pFieldName + "\"."); \ + } + + const char* mode_str; + const char* type_str; + ComponentType component_type; + SCompression_Open3DGC* ext_o3dgc = new SCompression_Open3DGC; + + MESH_READ_COMPRESSEDDATA_MEMBER("buffer", ext_o3dgc->Buffer); + MESH_READ_COMPRESSEDDATA_MEMBER("byteOffset", ext_o3dgc->Offset); + MESH_READ_COMPRESSEDDATA_MEMBER("componentType", component_type); + MESH_READ_COMPRESSEDDATA_MEMBER("type", type_str); + MESH_READ_COMPRESSEDDATA_MEMBER("count", ext_o3dgc->Count); + MESH_READ_COMPRESSEDDATA_MEMBER("mode", mode_str); + MESH_READ_COMPRESSEDDATA_MEMBER("indicesCount", ext_o3dgc->IndicesCount); + MESH_READ_COMPRESSEDDATA_MEMBER("verticesCount", ext_o3dgc->VerticesCount); + + #undef MESH_READ_COMPRESSEDDATA_MEMBER + + // Check some values + if(strcmp(type_str, "SCALAR")) throw DeadlyImportError("GLTF: only \"SCALAR\" type is supported for compressed data."); + if(component_type != ComponentType_UNSIGNED_BYTE) throw DeadlyImportError("GLTF: only \"UNSIGNED_BYTE\" component type is supported for compressed data."); + + // Set read/write data mode. + if(strcmp(mode_str, "binary") == 0) + ext_o3dgc->Binary = true; + else if(strcmp(mode_str, "ascii") == 0) + ext_o3dgc->Binary = false; + else + throw DeadlyImportError(std::string("GLTF: for compressed data supported modes is: \"ascii\", \"binary\". Not the: \"") + mode_str + "\"."); + + /************************ Decoding ************************/ + Decode_O3DGC(*ext_o3dgc, pAsset_Root); + Extension.push_back(ext_o3dgc);// store info in mesh extensions list. + }// if(it_memb->name.GetString() == "Open3DGC-compression") + else +#endif + { + throw DeadlyImportError(std::string("GLTF: Unknown mesh extension: \"") + it_memb->name.GetString() + "\"."); + } + }// for(Value::MemberIterator it_memb = json_extensions->MemberBegin(); it_memb != json_extensions->MemberEnd(); json_extensions++) + +mr_skip_extensions: + + return;// After label some operators must be present. +} + +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC +inline void Mesh::Decode_O3DGC(const SCompression_Open3DGC& pCompression_Open3DGC, Asset& pAsset_Root) +{ +typedef unsigned short IndicesType;///< \sa glTFExporter::ExportMeshes. + +o3dgc::SC3DMCDecoder decoder; +o3dgc::IndexedFaceSet ifs; +o3dgc::BinaryStream bstream; +uint8_t* decoded_data; +size_t decoded_data_size = 0; +Ref buf = pAsset_Root.buffers.Get(pCompression_Open3DGC.Buffer); + + // Read data from buffer and place it in BinaryStream for decoder. + // Just "Count" because always is used type equivalent to uint8_t. + bstream.LoadFromBuffer(&buf->GetPointer()[pCompression_Open3DGC.Offset], static_cast(pCompression_Open3DGC.Count)); + + // After decoding header we can get size of primitives. + if(decoder.DecodeHeader(ifs, bstream) != o3dgc::O3DGC_OK) throw DeadlyImportError("GLTF: can not decode Open3DGC header."); + + /****************** Get sizes of arrays and check sizes ******************/ + // Note. See "Limitations for meshes when using Open3DGC-compression". + + // Indices + size_t size_coordindex = ifs.GetNCoordIndex() * 3;// See float attributes note. + + if(primitives[0].indices->count != size_coordindex) + throw DeadlyImportError("GLTF: Open3DGC. Compressed indices count (" + std::to_string(size_coordindex) + + ") not equal to uncompressed (" + std::to_string(primitives[0].indices->count) + ")."); + + size_coordindex *= sizeof(IndicesType); + // Coordinates + size_t size_coord = ifs.GetNCoord();// See float attributes note. + + if(primitives[0].attributes.position[0]->count != size_coord) + throw DeadlyImportError("GLTF: Open3DGC. Compressed positions count (" + std::to_string(size_coord) + + ") not equal to uncompressed (" + std::to_string(primitives[0].attributes.position[0]->count) + ")."); + + size_coord *= 3 * sizeof(float); + // Normals + size_t size_normal = ifs.GetNNormal();// See float attributes note. + + if(primitives[0].attributes.normal[0]->count != size_normal) + throw DeadlyImportError("GLTF: Open3DGC. Compressed normals count (" + std::to_string(size_normal) + + ") not equal to uncompressed (" + std::to_string(primitives[0].attributes.normal[0]->count) + ")."); + + size_normal *= 3 * sizeof(float); + // Additional attributes. + std::vector size_floatattr; + std::vector size_intattr; + + size_floatattr.resize(ifs.GetNumFloatAttributes()); + size_intattr.resize(ifs.GetNumIntAttributes()); + + decoded_data_size = size_coordindex + size_coord + size_normal; + for(size_t idx = 0, idx_end = size_floatattr.size(), idx_texcoord = 0; idx < idx_end; idx++) + { + // size = number_of_elements * components_per_element * size_of_component. + // Note. But as you can see above, at first we are use this variable in meaning "count". After checking count of objects... + size_t tval = ifs.GetNFloatAttribute(static_cast(idx)); + + switch(ifs.GetFloatAttributeType(static_cast(idx))) + { + case o3dgc::O3DGC_IFS_FLOAT_ATTRIBUTE_TYPE_TEXCOORD: + // Check situation when encoded data contain texture coordinates but primitive not. + if(idx_texcoord < primitives[0].attributes.texcoord.size()) + { + if(primitives[0].attributes.texcoord[idx]->count != tval) + throw DeadlyImportError("GLTF: Open3DGC. Compressed texture coordinates count (" + std::to_string(tval) + + ") not equal to uncompressed (" + std::to_string(primitives[0].attributes.texcoord[idx]->count) + ")."); + + idx_texcoord++; + } + else + { + ifs.SetNFloatAttribute(static_cast(idx), 0ul);// Disable decoding this attribute. + } + + break; + default: + throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of float attribute: " + to_string(ifs.GetFloatAttributeType(static_cast(idx)))); + } + + tval *= ifs.GetFloatAttributeDim(static_cast(idx)) * sizeof(o3dgc::Real);// After checking count of objects we can get size of array. + size_floatattr[idx] = tval; + decoded_data_size += tval; + } + + for(size_t idx = 0, idx_end = size_intattr.size(); idx < idx_end; idx++) + { + // size = number_of_elements * components_per_element * size_of_component. See float attributes note. + size_t tval = ifs.GetNIntAttribute(static_cast(idx)); + switch( ifs.GetIntAttributeType(static_cast(idx) ) ) + { + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_UNKOWN: + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX: + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_JOINT_ID: + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX_BUFFER_ID: + break; + + default: + throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of int attribute: " + to_string(ifs.GetIntAttributeType(static_cast(idx)))); + } + + tval *= ifs.GetIntAttributeDim(static_cast(idx)) * sizeof(long);// See float attributes note. + size_intattr[idx] = tval; + decoded_data_size += tval; + } + + // Create array for decoded data. + decoded_data = new uint8_t[decoded_data_size]; + + /****************** Set right array regions for decoder ******************/ + + auto get_buf_offset = [](Ref& pAccessor) -> size_t { return pAccessor->byteOffset + pAccessor->bufferView->byteOffset; }; + + // Indices + ifs.SetCoordIndex((IndicesType* const)(decoded_data + get_buf_offset(primitives[0].indices))); + // Coordinates + ifs.SetCoord((o3dgc::Real* const)(decoded_data + get_buf_offset(primitives[0].attributes.position[0]))); + // Normals + if(size_normal) + { + ifs.SetNormal((o3dgc::Real* const)(decoded_data + get_buf_offset(primitives[0].attributes.normal[0]))); + } + + for(size_t idx = 0, idx_end = size_floatattr.size(), idx_texcoord = 0; idx < idx_end; idx++) + { + switch(ifs.GetFloatAttributeType(static_cast(idx))) + { + case o3dgc::O3DGC_IFS_FLOAT_ATTRIBUTE_TYPE_TEXCOORD: + if(idx_texcoord < primitives[0].attributes.texcoord.size()) + { + // See above about absent attributes. + ifs.SetFloatAttribute(static_cast(idx), (o3dgc::Real* const)(decoded_data + get_buf_offset(primitives[0].attributes.texcoord[idx]))); + idx_texcoord++; + } + + break; + default: + throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of float attribute: " + to_string(ifs.GetFloatAttributeType(static_cast(idx)))); + } + } + + for(size_t idx = 0, idx_end = size_intattr.size(); idx < idx_end; idx++) { + switch(ifs.GetIntAttributeType(static_cast(idx))) { + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_UNKOWN: + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX: + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_JOINT_ID: + case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX_BUFFER_ID: + break; + + // ifs.SetIntAttribute(idx, (long* const)(decoded_data + get_buf_offset(primitives[0].attributes.joint))); + default: + throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of int attribute: " + to_string(ifs.GetIntAttributeType(static_cast(idx)))); + } + } + + // + // Decode data + // + if ( decoder.DecodePayload( ifs, bstream ) != o3dgc::O3DGC_OK ) { + throw DeadlyImportError( "GLTF: can not decode Open3DGC data." ); + } + + // Set encoded region for "buffer". + buf->EncodedRegion_Mark(pCompression_Open3DGC.Offset, pCompression_Open3DGC.Count, decoded_data, decoded_data_size, id); + // No. Do not delete "output_data". After calling "EncodedRegion_Mark" bufferView is owner of "output_data". + // "delete [] output_data;" +} +#endif + +inline void Camera::Read(Value& obj, Asset& r) +{ + type = MemberOrDefault(obj, "type", Camera::Perspective); + + const char* subobjId = (type == Camera::Orthographic) ? "ortographic" : "perspective"; + + Value* it = FindObject(obj, subobjId); + if (!it) throw DeadlyImportError("GLTF: Camera missing its parameters"); + + if (type == Camera::Perspective) { + perspective.aspectRatio = MemberOrDefault(*it, "aspectRatio", 0.f); + perspective.yfov = MemberOrDefault(*it, "yfov", 3.1415f/2.f); + perspective.zfar = MemberOrDefault(*it, "zfar", 100.f); + perspective.znear = MemberOrDefault(*it, "znear", 0.01f); + } + else { + ortographic.xmag = MemberOrDefault(obj, "xmag", 1.f); + ortographic.ymag = MemberOrDefault(obj, "ymag", 1.f); + ortographic.zfar = MemberOrDefault(obj, "zfar", 100.f); + ortographic.znear = MemberOrDefault(obj, "znear", 0.01f); + } +} + +inline void Light::Read(Value& obj, Asset& r) +{ + SetDefaults(); + + if (Value* type = FindString(obj, "type")) { + const char* t = type->GetString(); + if (strcmp(t, "ambient") == 0) this->type = Type_ambient; + else if (strcmp(t, "directional") == 0) this->type = Type_directional; + else if (strcmp(t, "point") == 0) this->type = Type_point; + else if (strcmp(t, "spot") == 0) this->type = Type_spot; + + if (this->type != Type_undefined) { + if (Value* vals = FindString(obj, t)) { + ReadMember(*vals, "color", color); + + ReadMember(*vals, "constantAttenuation", constantAttenuation); + ReadMember(*vals, "linearAttenuation", linearAttenuation); + ReadMember(*vals, "quadraticAttenuation", quadraticAttenuation); + ReadMember(*vals, "distance", distance); + + ReadMember(*vals, "falloffAngle", falloffAngle); + ReadMember(*vals, "falloffExponent", falloffExponent); + } + } + } +} + +inline void Light::SetDefaults() +{ + #ifndef M_PI + const float M_PI = 3.14159265358979323846f; + #endif + + type = Type_undefined; + + SetVector(color, 0.f, 0.f, 0.f, 1.f); + + constantAttenuation = 0.f; + linearAttenuation = 1.f; + quadraticAttenuation = 1.f; + distance = 0.f; + + falloffAngle = static_cast(M_PI / 2.f); + falloffExponent = 0.f; +} + +inline void Node::Read(Value& obj, Asset& r) +{ + if (Value* children = FindArray(obj, "children")) { + this->children.reserve(children->Size()); + for (unsigned int i = 0; i < children->Size(); ++i) { + Value& child = (*children)[i]; + if (child.IsString()) { + // get/create the child node + Ref chn = r.nodes.Get(child.GetString()); + if (chn) this->children.push_back(chn); + } + } + } + + + if (Value* matrix = FindArray(obj, "matrix")) { + ReadValue(*matrix, this->matrix); + } + else { + ReadMember(obj, "translation", translation); + ReadMember(obj, "scale", scale); + ReadMember(obj, "rotation", rotation); + } + + if (Value* meshes = FindArray(obj, "meshes")) { + unsigned numMeshes = (unsigned)meshes->Size(); + + std::vector meshList; + + this->meshes.reserve(numMeshes); + for (unsigned i = 0; i < numMeshes; ++i) { + if ((*meshes)[i].IsString()) { + Ref mesh = r.meshes.Get((*meshes)[i].GetString()); + if (mesh) this->meshes.push_back(mesh); + } + } + } + + if (Value* camera = FindString(obj, "camera")) { + this->camera = r.cameras.Get(camera->GetString()); + if (this->camera) + this->camera->id = this->id; + } + + // TODO load "skeletons", "skin", "jointName" + + if (Value* extensions = FindObject(obj, "extensions")) { + if (r.extensionsUsed.KHR_materials_common) { + + if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { + if (Value* light = FindString(*ext, "light")) { + this->light = r.lights.Get(light->GetString()); + } + } + + } + } +} + +inline void Scene::Read(Value& obj, Asset& r) +{ + if (Value* array = FindArray(obj, "nodes")) { + for (unsigned int i = 0; i < array->Size(); ++i) { + if (!(*array)[i].IsString()) continue; + Ref node = r.nodes.Get((*array)[i].GetString()); + if (node) + this->nodes.push_back(node); + } + } +} + + +inline void AssetMetadata::Read(Document& doc) +{ + // read the version, etc. + int statedVersion = 0; + if (Value* obj = FindObject(doc, "asset")) { + ReadMember(*obj, "copyright", copyright); + ReadMember(*obj, "generator", generator); + + premultipliedAlpha = MemberOrDefault(*obj, "premultipliedAlpha", false); + statedVersion = MemberOrDefault(*obj, "version", 0); + + if (Value* profile = FindObject(*obj, "profile")) { + ReadMember(*profile, "api", this->profile.api); + ReadMember(*profile, "version", this->profile.version); + } + } + + version = std::max(statedVersion, version); + if (version == 0) { + // if missing version, we'll assume version 1... + version = 1; + } + + if (version != 1) { + char msg[128]; + ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %d", version); + throw DeadlyImportError(msg); + } +} + + + +// +// Asset methods implementation +// + +inline void Asset::ReadBinaryHeader(IOStream& stream) +{ + GLB_Header header; + if (stream.Read(&header, sizeof(header), 1) != 1) { + throw DeadlyImportError("GLTF: Unable to read the file header"); + } + + if (strncmp((char*)header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)) != 0) { + throw DeadlyImportError("GLTF: Invalid binary glTF file"); + } + + AI_SWAP4(header.version); + asset.version = header.version; + if (header.version != 1) { + throw DeadlyImportError("GLTF: Unsupported binary glTF version"); + } + + AI_SWAP4(header.sceneFormat); + if (header.sceneFormat != SceneFormat_JSON) { + throw DeadlyImportError("GLTF: Unsupported binary glTF scene format"); + } + + AI_SWAP4(header.length); + AI_SWAP4(header.sceneLength); + + mSceneLength = static_cast(header.sceneLength); + + mBodyOffset = sizeof(header)+mSceneLength; + mBodyOffset = (mBodyOffset + 3) & ~3; // Round up to next multiple of 4 + + mBodyLength = header.length - mBodyOffset; +} + +inline void Asset::Load(const std::string& pFile, bool isBinary) +{ + mCurrentAssetDir.clear(); + int pos = std::max(int(pFile.rfind('/')), int(pFile.rfind('\\'))); + if (pos != int(std::string::npos)) mCurrentAssetDir = pFile.substr(0, pos + 1); + + shared_ptr stream(OpenFile(pFile.c_str(), "rb", true)); + if (!stream) { + throw DeadlyImportError("GLTF: Could not open file for reading"); + } + + // is binary? then read the header + if (isBinary) { + SetAsBinary(); // also creates the body buffer + ReadBinaryHeader(*stream); + } + else { + mSceneLength = stream->FileSize(); + mBodyLength = 0; + } + + + // read the scene data + + std::vector sceneData(mSceneLength + 1); + sceneData[mSceneLength] = '\0'; + + if (stream->Read(&sceneData[0], 1, mSceneLength) != mSceneLength) { + throw DeadlyImportError("GLTF: Could not read the file contents"); + } + + + // parse the JSON document + + Document doc; + doc.ParseInsitu(&sceneData[0]); + + if (doc.HasParseError()) { + char buffer[32]; + ai_snprintf(buffer, 32, "%d", static_cast(doc.GetErrorOffset())); + throw DeadlyImportError(std::string("GLTF: JSON parse error, offset ") + buffer + ": " + + GetParseError_En(doc.GetParseError())); + } + + if (!doc.IsObject()) { + throw DeadlyImportError("GLTF: JSON document root must be a JSON object"); + } + + // Fill the buffer instance for the current file embedded contents + if (mBodyLength > 0) { + if (!mBodyBuffer->LoadFromStream(*stream, mBodyLength, mBodyOffset)) { + throw DeadlyImportError("GLTF: Unable to read gltf file"); + } + } + + + // Load the metadata + asset.Read(doc); + ReadExtensionsUsed(doc); + + // Prepare the dictionaries + for (size_t i = 0; i < mDicts.size(); ++i) { + mDicts[i]->AttachToDocument(doc); + } + + + + // Read the "scene" property, which specifies which scene to load + // and recursively load everything referenced by it + if (Value* scene = FindString(doc, "scene")) { + this->scene = scenes.Get(scene->GetString()); + } + + // Clean up + for (size_t i = 0; i < mDicts.size(); ++i) { + mDicts[i]->DetachFromDocument(); + } +} + +inline void Asset::SetAsBinary() +{ + if (!extensionsUsed.KHR_binary_glTF) { + extensionsUsed.KHR_binary_glTF = true; + mBodyBuffer = buffers.Create("binary_glTF"); + mBodyBuffer->MarkAsSpecial(); + } +} + + +inline void Asset::ReadExtensionsUsed(Document& doc) +{ + Value* extsUsed = FindArray(doc, "extensionsUsed"); + if (!extsUsed) return; + + std::gltf_unordered_map exts; + + for (unsigned int i = 0; i < extsUsed->Size(); ++i) { + if ((*extsUsed)[i].IsString()) { + exts[(*extsUsed)[i].GetString()] = true; + } + } + + #define CHECK_EXT(EXT) \ + if (exts.find(#EXT) != exts.end()) extensionsUsed.EXT = true; + + CHECK_EXT(KHR_binary_glTF); + CHECK_EXT(KHR_materials_common); + + #undef CHECK_EXT +} + +inline IOStream* Asset::OpenFile(std::string path, const char* mode, bool absolute) +{ + #ifdef ASSIMP_API + return mIOSystem->Open(path, mode); + #else + if (path.size() < 2) return 0; + if (!absolute && path[1] != ':' && path[0] != '/') { // relative? + path = mCurrentAssetDir + path; + } + FILE* f = fopen(path.c_str(), mode); + return f ? new IOStream(f) : 0; + #endif +} + +inline std::string Asset::FindUniqueID(const std::string& str, const char* suffix) +{ + std::string id = str; + + if (!id.empty()) { + if (mUsedIds.find(id) == mUsedIds.end()) + return id; + + id += "_"; + } + + id += suffix; + + Asset::IdMap::iterator it = mUsedIds.find(id); + if (it == mUsedIds.end()) + return id; + + char buffer[256]; + int offset = ai_snprintf(buffer, sizeof(buffer), "%s_", id.c_str()); + for (int i = 0; it != mUsedIds.end(); ++i) { + ai_snprintf(buffer + offset, sizeof(buffer) - offset, "%d", i); + id = buffer; + it = mUsedIds.find(id); + } + + return id; +} + +namespace Util { + + inline + bool ParseDataURI(const char* const_uri, size_t uriLen, DataURI& out) { + if ( NULL == const_uri ) { + return false; + } + + if (const_uri[0] != 0x10) { // we already parsed this uri? + if (strncmp(const_uri, "data:", 5) != 0) // not a data uri? + return false; + } + + // set defaults + out.mediaType = "text/plain"; + out.charset = "US-ASCII"; + out.base64 = false; + + char* uri = const_cast(const_uri); + if (uri[0] != 0x10) { + uri[0] = 0x10; + uri[1] = uri[2] = uri[3] = uri[4] = 0; + + size_t i = 5, j; + if (uri[i] != ';' && uri[i] != ',') { // has media type? + uri[1] = char(i); + for (; uri[i] != ';' && uri[i] != ',' && i < uriLen; ++i) { + // nothing to do! + } + } + while (uri[i] == ';' && i < uriLen) { + uri[i++] = '\0'; + for (j = i; uri[i] != ';' && uri[i] != ',' && i < uriLen; ++i) { + // nothing to do! + } + + if ( strncmp( uri + j, "charset=", 8 ) == 0 ) { + uri[2] = char(j + 8); + } else if ( strncmp( uri + j, "base64", 6 ) == 0 ) { + uri[3] = char(j); + } + } + if (i < uriLen) { + uri[i++] = '\0'; + uri[4] = char(i); + } else { + uri[1] = uri[2] = uri[3] = 0; + uri[4] = 5; + } + } + + if ( uri[ 1 ] != 0 ) { + out.mediaType = uri + uri[ 1 ]; + } + if ( uri[ 2 ] != 0 ) { + out.charset = uri + uri[ 2 ]; + } + if ( uri[ 3 ] != 0 ) { + out.base64 = true; + } + out.data = uri + uri[4]; + out.dataLength = (uri + uriLen) - out.data; + + return true; + } + + template + struct DATA + { + static const uint8_t tableDecodeBase64[128]; + }; + + template + const uint8_t DATA::tableDecodeBase64[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 64, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0 + }; + + inline char EncodeCharBase64(uint8_t b) + { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="[size_t(b)]; + } + + inline uint8_t DecodeCharBase64(char c) + { + return DATA::tableDecodeBase64[size_t(c)]; // TODO faster with lookup table or ifs? + /*if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return 64; // '-' */ + } + + inline size_t DecodeBase64(const char* in, size_t inLength, uint8_t*& out) + { + ai_assert(inLength % 4 == 0); + + if (inLength < 4) { + out = 0; + return 0; + } + + int nEquals = int(in[inLength - 1] == '=') + + int(in[inLength - 2] == '='); + + size_t outLength = (inLength * 3) / 4 - nEquals; + out = new uint8_t[outLength]; + memset(out, 0, outLength); + + size_t i, j = 0; + + for (i = 0; i + 4 < inLength; i += 4) { + uint8_t b0 = DecodeCharBase64(in[i]); + uint8_t b1 = DecodeCharBase64(in[i + 1]); + uint8_t b2 = DecodeCharBase64(in[i + 2]); + uint8_t b3 = DecodeCharBase64(in[i + 3]); + + out[j++] = (uint8_t)((b0 << 2) | (b1 >> 4)); + out[j++] = (uint8_t)((b1 << 4) | (b2 >> 2)); + out[j++] = (uint8_t)((b2 << 6) | b3); + } + + { + uint8_t b0 = DecodeCharBase64(in[i]); + uint8_t b1 = DecodeCharBase64(in[i + 1]); + uint8_t b2 = DecodeCharBase64(in[i + 2]); + uint8_t b3 = DecodeCharBase64(in[i + 3]); + + out[j++] = (uint8_t)((b0 << 2) | (b1 >> 4)); + if (b2 < 64) out[j++] = (uint8_t)((b1 << 4) | (b2 >> 2)); + if (b3 < 64) out[j++] = (uint8_t)((b2 << 6) | b3); + } + + return outLength; + } + + + + inline void EncodeBase64( + const uint8_t* in, size_t inLength, + std::string& out) + { + size_t outLength = ((inLength + 2) / 3) * 4; + + size_t j = out.size(); + out.resize(j + outLength); + + for (size_t i = 0; i < inLength; i += 3) { + uint8_t b = (in[i] & 0xFC) >> 2; + out[j++] = EncodeCharBase64(b); + + b = (in[i] & 0x03) << 4; + if (i + 1 < inLength) { + b |= (in[i + 1] & 0xF0) >> 4; + out[j++] = EncodeCharBase64(b); + + b = (in[i + 1] & 0x0F) << 2; + if (i + 2 < inLength) { + b |= (in[i + 2] & 0xC0) >> 6; + out[j++] = EncodeCharBase64(b); + + b = in[i + 2] & 0x3F; + out[j++] = EncodeCharBase64(b); + } + else { + out[j++] = EncodeCharBase64(b); + out[j++] = '='; + } + } + else { + out[j++] = EncodeCharBase64(b); + out[j++] = '='; + out[j++] = '='; + } + } + } + +} + +} // ns glTF diff --git a/code/glTF2AssetWriter.h b/code/glTF2AssetWriter.h new file mode 100644 index 000000000..186d32a15 --- /dev/null +++ b/code/glTF2AssetWriter.h @@ -0,0 +1,95 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +/** @file glTFWriter.h + * Declares a class to write gltf/glb files + * + * glTF Extensions Support: + * KHR_binary_glTF: full + * KHR_materials_common: full + */ +#ifndef GLTFASSETWRITER_H_INC +#define GLTFASSETWRITER_H_INC + +#ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER + +#include "glTFAsset.h" + +namespace glTF +{ + +using rapidjson::MemoryPoolAllocator; + +class AssetWriter +{ + template + friend void WriteLazyDict(LazyDict& d, AssetWriter& w); + +private: + + void WriteBinaryData(IOStream* outfile, size_t sceneLength); + + void WriteMetadata(); + void WriteExtensionsUsed(); + + template + void WriteObjects(LazyDict& d); + +public: + Document mDoc; + Asset& mAsset; + + MemoryPoolAllocator<>& mAl; + + AssetWriter(Asset& asset); + + void WriteFile(const char* path); + void WriteGLBFile(const char* path); +}; + +} + +// Include the implementation of the methods +#include "glTFAssetWriter.inl" + +#endif // ASSIMP_BUILD_NO_GLTF_IMPORTER + +#endif // GLTFASSETWRITER_H_INC diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl new file mode 100644 index 000000000..d3a553c27 --- /dev/null +++ b/code/glTF2AssetWriter.inl @@ -0,0 +1,685 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +#include +#include +#include + +namespace glTF { + + using rapidjson::StringBuffer; + using rapidjson::PrettyWriter; + using rapidjson::Writer; + using rapidjson::StringRef; + using rapidjson::StringRef; + + namespace { + + template + inline Value& MakeValue(Value& val, float(&r)[N], MemoryPoolAllocator<>& al) { + val.SetArray(); + val.Reserve(N, al); + for (decltype(N) i = 0; i < N; ++i) { + val.PushBack(r[i], al); + } + return val; + } + + inline Value& MakeValue(Value& val, const std::vector & r, MemoryPoolAllocator<>& al) { + val.SetArray(); + val.Reserve(static_cast(r.size()), al); + for (unsigned int i = 0; i < r.size(); ++i) { + val.PushBack(r[i], al); + } + return val; + } + + template + inline void AddRefsVector(Value& obj, const char* fieldId, std::vector< Ref >& v, MemoryPoolAllocator<>& al) { + if (v.empty()) return; + Value lst; + lst.SetArray(); + lst.Reserve(unsigned(v.size()), al); + for (size_t i = 0; i < v.size(); ++i) { + lst.PushBack(StringRef(v[i]->id), al); + } + obj.AddMember(StringRef(fieldId), lst, al); + } + + + } + + inline void Write(Value& obj, Accessor& a, AssetWriter& w) + { + obj.AddMember("bufferView", Value(a.bufferView->id, w.mAl).Move(), w.mAl); + obj.AddMember("byteOffset", a.byteOffset, w.mAl); + obj.AddMember("byteStride", a.byteStride, w.mAl); + obj.AddMember("componentType", int(a.componentType), w.mAl); + obj.AddMember("count", a.count, w.mAl); + obj.AddMember("type", StringRef(AttribType::ToString(a.type)), w.mAl); + + Value vTmpMax, vTmpMin; + obj.AddMember("max", MakeValue(vTmpMax, a.max, w.mAl), w.mAl); + obj.AddMember("min", MakeValue(vTmpMin, a.min, w.mAl), w.mAl); + } + + inline void Write(Value& obj, Animation& a, AssetWriter& w) + { + /****************** Channels *******************/ + Value channels; + channels.SetArray(); + channels.Reserve(unsigned(a.Channels.size()), w.mAl); + + for (size_t i = 0; i < unsigned(a.Channels.size()); ++i) { + Animation::AnimChannel& c = a.Channels[i]; + Value valChannel; + valChannel.SetObject(); + { + valChannel.AddMember("sampler", c.sampler, w.mAl); + + Value valTarget; + valTarget.SetObject(); + { + valTarget.AddMember("id", StringRef(c.target.id->id), w.mAl); + valTarget.AddMember("path", c.target.path, w.mAl); + } + valChannel.AddMember("target", valTarget, w.mAl); + } + channels.PushBack(valChannel, w.mAl); + } + obj.AddMember("channels", channels, w.mAl); + + /****************** Parameters *******************/ + Value valParameters; + valParameters.SetObject(); + { + if (a.Parameters.TIME) { + valParameters.AddMember("TIME", StringRef(a.Parameters.TIME->id), w.mAl); + } + if (a.Parameters.rotation) { + valParameters.AddMember("rotation", StringRef(a.Parameters.rotation->id), w.mAl); + } + if (a.Parameters.scale) { + valParameters.AddMember("scale", StringRef(a.Parameters.scale->id), w.mAl); + } + if (a.Parameters.translation) { + valParameters.AddMember("translation", StringRef(a.Parameters.translation->id), w.mAl); + } + } + obj.AddMember("parameters", valParameters, w.mAl); + + /****************** Samplers *******************/ + Value valSamplers; + valSamplers.SetObject(); + + for (size_t i = 0; i < unsigned(a.Samplers.size()); ++i) { + Animation::AnimSampler& s = a.Samplers[i]; + Value valSampler; + valSampler.SetObject(); + { + valSampler.AddMember("input", s.input, w.mAl); + valSampler.AddMember("interpolation", s.interpolation, w.mAl); + valSampler.AddMember("output", s.output, w.mAl); + } + valSamplers.AddMember(StringRef(s.id), valSampler, w.mAl); + } + obj.AddMember("samplers", valSamplers, w.mAl); + } + + inline void Write(Value& obj, Buffer& b, AssetWriter& w) + { + const char* type; + switch (b.type) { + case Buffer::Type_text: + type = "text"; break; + default: + type = "arraybuffer"; + } + + obj.AddMember("byteLength", static_cast(b.byteLength), w.mAl); + obj.AddMember("type", StringRef(type), w.mAl); + obj.AddMember("uri", Value(b.GetURI(), w.mAl).Move(), w.mAl); + } + + inline void Write(Value& obj, BufferView& bv, AssetWriter& w) + { + obj.AddMember("buffer", Value(bv.buffer->id, w.mAl).Move(), w.mAl); + obj.AddMember("byteOffset", static_cast(bv.byteOffset), w.mAl); + obj.AddMember("byteLength", static_cast(bv.byteLength), w.mAl); + obj.AddMember("target", int(bv.target), w.mAl); + } + + inline void Write(Value& obj, Camera& c, AssetWriter& w) + { + + } + + inline void Write(Value& obj, Image& img, AssetWriter& w) + { + std::string uri; + if (w.mAsset.extensionsUsed.KHR_binary_glTF && img.bufferView) { + Value exts, ext; + exts.SetObject(); + ext.SetObject(); + + ext.AddMember("bufferView", StringRef(img.bufferView->id), w.mAl); + + if (!img.mimeType.empty()) + ext.AddMember("mimeType", StringRef(img.mimeType), w.mAl); + + exts.AddMember("KHR_binary_glTF", ext, w.mAl); + obj.AddMember("extensions", exts, w.mAl); + return; + } + else if (img.HasData()) { + uri = "data:" + (img.mimeType.empty() ? "application/octet-stream" : img.mimeType); + uri += ";base64,"; + Util::EncodeBase64(img.GetData(), img.GetDataLength(), uri); + } + else { + uri = img.uri; + } + + obj.AddMember("uri", Value(uri, w.mAl).Move(), w.mAl); + } + + namespace { + inline void WriteColorOrTex(Value& obj, TexProperty& prop, const char* propName, MemoryPoolAllocator<>& al) + { + if (prop.texture) + obj.AddMember(StringRef(propName), Value(prop.texture->id, al).Move(), al); + else { + Value col; + obj.AddMember(StringRef(propName), MakeValue(col, prop.color, al), al); + } + } + } + + inline void Write(Value& obj, Material& m, AssetWriter& w) + { + Value v; + v.SetObject(); + { + WriteColorOrTex(v, m.ambient, "ambient", w.mAl); + WriteColorOrTex(v, m.diffuse, "diffuse", w.mAl); + WriteColorOrTex(v, m.specular, "specular", w.mAl); + WriteColorOrTex(v, m.emission, "emission", w.mAl); + + if (m.transparent) + v.AddMember("transparency", m.transparency, w.mAl); + + v.AddMember("shininess", m.shininess, w.mAl); + } + obj.AddMember("values", v, w.mAl); + } + + namespace { + inline void WriteAttrs(AssetWriter& w, Value& attrs, Mesh::AccessorList& lst, + const char* semantic, bool forceNumber = false) + { + if (lst.empty()) return; + if (lst.size() == 1 && !forceNumber) { + attrs.AddMember(StringRef(semantic), Value(lst[0]->id, w.mAl).Move(), w.mAl); + } + else { + for (size_t i = 0; i < lst.size(); ++i) { + char buffer[32]; + ai_snprintf(buffer, 32, "%s_%d", semantic, int(i)); + attrs.AddMember(Value(buffer, w.mAl).Move(), Value(lst[i]->id, w.mAl).Move(), w.mAl); + } + } + } + } + + inline void Write(Value& obj, Mesh& m, AssetWriter& w) + { + /********************* Name **********************/ + obj.AddMember("name", m.name, w.mAl); + + /**************** Mesh extensions ****************/ + if(m.Extension.size() > 0) + { + Value json_extensions; + + json_extensions.SetObject(); + for(Mesh::SExtension* ptr_ext : m.Extension) + { + switch(ptr_ext->Type) + { +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + case Mesh::SExtension::EType::Compression_Open3DGC: + { + Value json_comp_data; + Mesh::SCompression_Open3DGC* ptr_ext_comp = (Mesh::SCompression_Open3DGC*)ptr_ext; + + // filling object "compressedData" + json_comp_data.SetObject(); + json_comp_data.AddMember("buffer", ptr_ext_comp->Buffer, w.mAl); + json_comp_data.AddMember("byteOffset", ptr_ext_comp->Offset, w.mAl); + json_comp_data.AddMember("componentType", 5121, w.mAl); + json_comp_data.AddMember("type", "SCALAR", w.mAl); + json_comp_data.AddMember("count", ptr_ext_comp->Count, w.mAl); + if(ptr_ext_comp->Binary) + json_comp_data.AddMember("mode", "binary", w.mAl); + else + json_comp_data.AddMember("mode", "ascii", w.mAl); + + json_comp_data.AddMember("indicesCount", ptr_ext_comp->IndicesCount, w.mAl); + json_comp_data.AddMember("verticesCount", ptr_ext_comp->VerticesCount, w.mAl); + // filling object "Open3DGC-compression" + Value json_o3dgc; + + json_o3dgc.SetObject(); + json_o3dgc.AddMember("compressedData", json_comp_data, w.mAl); + // add member to object "extensions" + json_extensions.AddMember("Open3DGC-compression", json_o3dgc, w.mAl); + } + + break; +#endif + default: + throw DeadlyImportError("GLTF: Can not write mesh: unknown mesh extension, only Open3DGC is supported."); + }// switch(ptr_ext->Type) + }// for(Mesh::SExtension* ptr_ext : m.Extension) + + // Add extensions to mesh + obj.AddMember("extensions", json_extensions, w.mAl); + }// if(m.Extension.size() > 0) + + /****************** Primitives *******************/ + Value primitives; + primitives.SetArray(); + primitives.Reserve(unsigned(m.primitives.size()), w.mAl); + + for (size_t i = 0; i < m.primitives.size(); ++i) { + Mesh::Primitive& p = m.primitives[i]; + Value prim; + prim.SetObject(); + { + prim.AddMember("mode", Value(int(p.mode)).Move(), w.mAl); + + if (p.material) + prim.AddMember("material", p.material->id, w.mAl); + + if (p.indices) + prim.AddMember("indices", Value(p.indices->id, w.mAl).Move(), w.mAl); + + Value attrs; + attrs.SetObject(); + { + WriteAttrs(w, attrs, p.attributes.position, "POSITION"); + WriteAttrs(w, attrs, p.attributes.normal, "NORMAL"); + WriteAttrs(w, attrs, p.attributes.texcoord, "TEXCOORD", true); + WriteAttrs(w, attrs, p.attributes.color, "COLOR"); + WriteAttrs(w, attrs, p.attributes.joint, "JOINT"); + WriteAttrs(w, attrs, p.attributes.jointmatrix, "JOINTMATRIX"); + WriteAttrs(w, attrs, p.attributes.weight, "WEIGHT"); + } + prim.AddMember("attributes", attrs, w.mAl); + } + primitives.PushBack(prim, w.mAl); + } + + obj.AddMember("primitives", primitives, w.mAl); + } + + inline void Write(Value& obj, Node& n, AssetWriter& w) + { + + if (n.matrix.isPresent) { + Value val; + obj.AddMember("matrix", MakeValue(val, n.matrix.value, w.mAl).Move(), w.mAl); + } + + if (n.translation.isPresent) { + Value val; + obj.AddMember("translation", MakeValue(val, n.translation.value, w.mAl).Move(), w.mAl); + } + + if (n.scale.isPresent) { + Value val; + obj.AddMember("scale", MakeValue(val, n.scale.value, w.mAl).Move(), w.mAl); + } + if (n.rotation.isPresent) { + Value val; + obj.AddMember("rotation", MakeValue(val, n.rotation.value, w.mAl).Move(), w.mAl); + } + + AddRefsVector(obj, "children", n.children, w.mAl); + + AddRefsVector(obj, "meshes", n.meshes, w.mAl); + + AddRefsVector(obj, "skeletons", n.skeletons, w.mAl); + + if (n.skin) { + obj.AddMember("skin", Value(n.skin->id, w.mAl).Move(), w.mAl); + } + + if (!n.jointName.empty()) { + obj.AddMember("jointName", n.jointName, w.mAl); + } + } + + inline void Write(Value& obj, Program& b, AssetWriter& w) + { + + } + + inline void Write(Value& obj, Sampler& b, AssetWriter& w) + { + if (b.wrapS) { + obj.AddMember("wrapS", b.wrapS, w.mAl); + } + if (b.wrapT) { + obj.AddMember("wrapT", b.wrapT, w.mAl); + } + if (b.magFilter) { + obj.AddMember("magFilter", b.magFilter, w.mAl); + } + if (b.minFilter) { + obj.AddMember("minFilter", b.minFilter, w.mAl); + } + } + + inline void Write(Value& scene, Scene& s, AssetWriter& w) + { + AddRefsVector(scene, "nodes", s.nodes, w.mAl); + } + + inline void Write(Value& obj, Shader& b, AssetWriter& w) + { + + } + + inline void Write(Value& obj, Skin& b, AssetWriter& w) + { + /****************** jointNames *******************/ + Value vJointNames; + vJointNames.SetArray(); + vJointNames.Reserve(unsigned(b.jointNames.size()), w.mAl); + + for (size_t i = 0; i < unsigned(b.jointNames.size()); ++i) { + vJointNames.PushBack(StringRef(b.jointNames[i]->jointName), w.mAl); + } + obj.AddMember("jointNames", vJointNames, w.mAl); + + if (b.bindShapeMatrix.isPresent) { + Value val; + obj.AddMember("bindShapeMatrix", MakeValue(val, b.bindShapeMatrix.value, w.mAl).Move(), w.mAl); + } + + if (b.inverseBindMatrices) { + obj.AddMember("inverseBindMatrices", Value(b.inverseBindMatrices->id, w.mAl).Move(), w.mAl); + } + + } + + inline void Write(Value& obj, Technique& b, AssetWriter& w) + { + + } + + inline void Write(Value& obj, Texture& tex, AssetWriter& w) + { + if (tex.source) { + obj.AddMember("source", Value(tex.source->id, w.mAl).Move(), w.mAl); + } + if (tex.sampler) { + obj.AddMember("sampler", Value(tex.sampler->id, w.mAl).Move(), w.mAl); + } + } + + inline void Write(Value& obj, Light& b, AssetWriter& w) + { + + } + + + inline AssetWriter::AssetWriter(Asset& a) + : mDoc() + , mAsset(a) + , mAl(mDoc.GetAllocator()) + { + mDoc.SetObject(); + + WriteMetadata(); + WriteExtensionsUsed(); + + // Dump the contents of the dictionaries + for (size_t i = 0; i < a.mDicts.size(); ++i) { + a.mDicts[i]->WriteObjects(*this); + } + + // Add the target scene field + if (mAsset.scene) { + mDoc.AddMember("scene", StringRef(mAsset.scene->id), mAl); + } + } + + inline void AssetWriter::WriteFile(const char* path) + { + std::unique_ptr jsonOutFile(mAsset.OpenFile(path, "wt", true)); + + if (jsonOutFile == 0) { + throw DeadlyExportError("Could not open output file: " + std::string(path)); + } + + StringBuffer docBuffer; + + PrettyWriter writer(docBuffer); + mDoc.Accept(writer); + + if (jsonOutFile->Write(docBuffer.GetString(), docBuffer.GetSize(), 1) != 1) { + throw DeadlyExportError("Failed to write scene data!"); + } + + // Write buffer data to separate .bin files + for (unsigned int i = 0; i < mAsset.buffers.Size(); ++i) { + Ref b = mAsset.buffers.Get(i); + + std::string binPath = b->GetURI(); + + std::unique_ptr binOutFile(mAsset.OpenFile(binPath, "wb", true)); + + if (binOutFile == 0) { + throw DeadlyExportError("Could not open output file: " + binPath); + } + + if (b->byteLength > 0) { + if (binOutFile->Write(b->GetPointer(), b->byteLength, 1) != 1) { + throw DeadlyExportError("Failed to write binary file: " + binPath); + } + } + } + } + + inline void AssetWriter::WriteGLBFile(const char* path) + { + std::unique_ptr outfile(mAsset.OpenFile(path, "wb", true)); + + if (outfile == 0) { + throw DeadlyExportError("Could not open output file: " + std::string(path)); + } + + // we will write the header later, skip its size + outfile->Seek(sizeof(GLB_Header), aiOrigin_SET); + + StringBuffer docBuffer; + Writer writer(docBuffer); + mDoc.Accept(writer); + + if (outfile->Write(docBuffer.GetString(), docBuffer.GetSize(), 1) != 1) { + throw DeadlyExportError("Failed to write scene data!"); + } + + WriteBinaryData(outfile.get(), docBuffer.GetSize()); + } + + inline void AssetWriter::WriteBinaryData(IOStream* outfile, size_t sceneLength) + { + // + // write the body data + // + + size_t bodyLength = 0; + if (Ref b = mAsset.GetBodyBuffer()) { + bodyLength = b->byteLength; + + if (bodyLength > 0) { + size_t bodyOffset = sizeof(GLB_Header) + sceneLength; + bodyOffset = (bodyOffset + 3) & ~3; // Round up to next multiple of 4 + + outfile->Seek(bodyOffset, aiOrigin_SET); + + if (outfile->Write(b->GetPointer(), b->byteLength, 1) != 1) { + throw DeadlyExportError("Failed to write body data!"); + } + } + } + + // + // write the header + // + + GLB_Header header; + memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); + + header.version = 1; + AI_SWAP4(header.version); + + header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); + AI_SWAP4(header.length); + + header.sceneLength = uint32_t(sceneLength); + AI_SWAP4(header.sceneLength); + + header.sceneFormat = SceneFormat_JSON; + AI_SWAP4(header.sceneFormat); + + outfile->Seek(0, aiOrigin_SET); + + if (outfile->Write(&header, 1, sizeof(header)) != sizeof(header)) { + throw DeadlyExportError("Failed to write the header!"); + } + } + + + inline void AssetWriter::WriteMetadata() + { + Value asset; + asset.SetObject(); + { + char versionChar[10]; + ai_snprintf(versionChar, sizeof(versionChar), "%d", mAsset.asset.version); + asset.AddMember("version", Value(versionChar, mAl).Move(), mAl); + + asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); + } + mDoc.AddMember("asset", asset, mAl); + } + + inline void AssetWriter::WriteExtensionsUsed() + { + Value exts; + exts.SetArray(); + { + if (false) + exts.PushBack(StringRef("KHR_binary_glTF"), mAl); + + if (false) + exts.PushBack(StringRef("KHR_materials_common"), mAl); + } + + if (!exts.Empty()) + mDoc.AddMember("extensionsUsed", exts, mAl); + } + + template + void AssetWriter::WriteObjects(LazyDict& d) + { + if (d.mObjs.empty()) return; + + Value* container = &mDoc; + + if (d.mExtId) { + Value* exts = FindObject(mDoc, "extensions"); + if (!exts) { + mDoc.AddMember("extensions", Value().SetObject().Move(), mDoc.GetAllocator()); + exts = FindObject(mDoc, "extensions"); + } + + if (!(container = FindObject(*exts, d.mExtId))) { + exts->AddMember(StringRef(d.mExtId), Value().SetObject().Move(), mDoc.GetAllocator()); + container = FindObject(*exts, d.mExtId); + } + } + + Value* dict; + if (!(dict = FindObject(*container, d.mDictId))) { + container->AddMember(StringRef(d.mDictId), Value().SetObject().Move(), mDoc.GetAllocator()); + dict = FindObject(*container, d.mDictId); + } + + for (size_t i = 0; i < d.mObjs.size(); ++i) { + if (d.mObjs[i]->IsSpecial()) continue; + + Value obj; + obj.SetObject(); + + if (!d.mObjs[i]->name.empty()) { + obj.AddMember("name", StringRef(d.mObjs[i]->name.c_str()), mAl); + } + + Write(obj, *d.mObjs[i], *this); + + dict->AddMember(StringRef(d.mObjs[i]->id), obj, mAl); + } + } + + template + void WriteLazyDict(LazyDict& d, AssetWriter& w) + { + w.WriteObjects(d); + } + +} + + diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp new file mode 100644 index 000000000..aeb5069aa --- /dev/null +++ b/code/glTF2Exporter.cpp @@ -0,0 +1,1007 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ +#ifndef ASSIMP_BUILD_NO_EXPORT +#ifndef ASSIMP_BUILD_NO_GLTF_EXPORTER + +#include "glTFExporter.h" + +#include "Exceptional.h" +#include "StringComparison.h" +#include "ByteSwapper.h" + +#include "SplitLargeMeshes.h" + +#include +#include +#include +#include +#include +#include + +// Header files, standart library. +#include +#include + +#include "glTFAssetWriter.h" + +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + // Header files, Open3DGC. +# include +#endif + +using namespace rapidjson; + +using namespace Assimp; +using namespace glTF; + +namespace Assimp { + + // ------------------------------------------------------------------------------------------------ + // Worker function for exporting a scene to GLTF. Prototyped and registered in Exporter.cpp + void ExportSceneGLTF(const char* pFile, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties) + { + // invoke the exporter + glTFExporter exporter(pFile, pIOSystem, pScene, pProperties, false); + } + + // ------------------------------------------------------------------------------------------------ + // Worker function for exporting a scene to GLB. Prototyped and registered in Exporter.cpp + void ExportSceneGLB(const char* pFile, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties) + { + // invoke the exporter + glTFExporter exporter(pFile, pIOSystem, pScene, pProperties, true); + } + +} // end of namespace Assimp + +glTFExporter::glTFExporter(const char* filename, IOSystem* pIOSystem, const aiScene* pScene, + const ExportProperties* pProperties, bool isBinary) + : mFilename(filename) + , mIOSystem(pIOSystem) + , mProperties(pProperties) +{ + aiScene* sceneCopy_tmp; + SceneCombiner::CopyScene(&sceneCopy_tmp, pScene); + std::unique_ptr sceneCopy(sceneCopy_tmp); + + SplitLargeMeshesProcess_Triangle tri_splitter; + tri_splitter.SetLimit(0xffff); + tri_splitter.Execute(sceneCopy.get()); + + SplitLargeMeshesProcess_Vertex vert_splitter; + vert_splitter.SetLimit(0xffff); + vert_splitter.Execute(sceneCopy.get()); + + mScene = sceneCopy.get(); + + mAsset.reset( new glTF::Asset( pIOSystem ) ); + + if (isBinary) { + mAsset->SetAsBinary(); + } + + ExportMetadata(); + + //for (unsigned int i = 0; i < pScene->mNumCameras; ++i) {} + + //for (unsigned int i = 0; i < pScene->mNumLights; ++i) {} + + ExportMaterials(); + + if (mScene->mRootNode) { + ExportNodeHierarchy(mScene->mRootNode); + } + + ExportMeshes(); + + //for (unsigned int i = 0; i < pScene->mNumTextures; ++i) {} + + ExportScene(); + + ExportAnimations(); + + glTF::AssetWriter writer(*mAsset); + + if (isBinary) { + writer.WriteGLBFile(filename); + } else { + writer.WriteFile(filename); + } +} + +/* + * Copy a 4x4 matrix from struct aiMatrix to typedef mat4. + * Also converts from row-major to column-major storage. + */ +static void CopyValue(const aiMatrix4x4& v, glTF::mat4& o) +{ + o[ 0] = v.a1; o[ 1] = v.b1; o[ 2] = v.c1; o[ 3] = v.d1; + o[ 4] = v.a2; o[ 5] = v.b2; o[ 6] = v.c2; o[ 7] = v.d2; + o[ 8] = v.a3; o[ 9] = v.b3; o[10] = v.c3; o[11] = v.d3; + o[12] = v.a4; o[13] = v.b4; o[14] = v.c4; o[15] = v.d4; +} + +static void CopyValue(const aiMatrix4x4& v, aiMatrix4x4& o) +{ + o.a1 = v.a1; o.a2 = v.a2; o.a3 = v.a3; o.a4 = v.a4; + o.b1 = v.b1; o.b2 = v.b2; o.b3 = v.b3; o.b4 = v.b4; + o.c1 = v.c1; o.c2 = v.c2; o.c3 = v.c3; o.c4 = v.c4; + o.d1 = v.d1; o.d2 = v.d2; o.d3 = v.d3; o.d4 = v.d4; +} + +static void IdentityMatrix4(glTF::mat4& o) +{ + o[ 0] = 1; o[ 1] = 0; o[ 2] = 0; o[ 3] = 0; + o[ 4] = 0; o[ 5] = 1; o[ 6] = 0; o[ 7] = 0; + o[ 8] = 0; o[ 9] = 0; o[10] = 1; o[11] = 0; + o[12] = 0; o[13] = 0; o[14] = 0; o[15] = 1; +} + +inline Ref ExportData(Asset& a, std::string& meshName, Ref& buffer, + unsigned int count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false) +{ + if (!count || !data) return Ref(); + + unsigned int numCompsIn = AttribType::GetNumComponents(typeIn); + unsigned int numCompsOut = AttribType::GetNumComponents(typeOut); + unsigned int bytesPerComp = ComponentTypeSize(compType); + + size_t offset = buffer->byteLength; + // make sure offset is correctly byte-aligned, as required by spec + size_t padding = offset % bytesPerComp; + offset += padding; + size_t length = count * numCompsOut * bytesPerComp; + buffer->Grow(length + padding); + + // bufferView + Ref bv = a.bufferViews.Create(a.FindUniqueID(meshName, "view")); + bv->buffer = buffer; + bv->byteOffset = unsigned(offset); + bv->byteLength = length; //! The target that the WebGL buffer should be bound to. + bv->target = isIndices ? BufferViewTarget_ELEMENT_ARRAY_BUFFER : BufferViewTarget_ARRAY_BUFFER; + + // accessor + Ref acc = a.accessors.Create(a.FindUniqueID(meshName, "accessor")); + acc->bufferView = bv; + acc->byteOffset = 0; + acc->byteStride = 0; + acc->componentType = compType; + acc->count = count; + acc->type = typeOut; + + // calculate min and max values + { + // Allocate and initialize with large values. + float float_MAX = 10000000000000.0f; + for (unsigned int i = 0 ; i < numCompsOut ; i++) { + acc->min.push_back( float_MAX); + acc->max.push_back(-float_MAX); + } + + // Search and set extreme values. + float valueTmp; + for (unsigned int i = 0 ; i < count ; i++) { + for (unsigned int j = 0 ; j < numCompsOut ; j++) { + if (numCompsOut == 1) { + valueTmp = static_cast(data)[i]; + } else { + valueTmp = static_cast(data)[i][j]; + } + + if (valueTmp < acc->min[j]) { + acc->min[j] = valueTmp; + } + if (valueTmp > acc->max[j]) { + acc->max[j] = valueTmp; + } + } + } + } + + // copy the data + acc->WriteData(count, data, numCompsIn*bytesPerComp); + + return acc; +} + +namespace { + void GetMatScalar(const aiMaterial* mat, float& val, const char* propName, int type, int idx) { + if (mat->Get(propName, type, idx, val) == AI_SUCCESS) {} + } +} + +void glTFExporter::GetTexSampler(const aiMaterial* mat, glTF::TexProperty& prop) +{ + std::string samplerId = mAsset->FindUniqueID("", "sampler"); + prop.texture->sampler = mAsset->samplers.Create(samplerId); + + aiTextureMapMode mapU, mapV; + aiGetMaterialInteger(mat,AI_MATKEY_MAPPINGMODE_U_DIFFUSE(0),(int*)&mapU); + aiGetMaterialInteger(mat,AI_MATKEY_MAPPINGMODE_V_DIFFUSE(0),(int*)&mapV); + + switch (mapU) { + case aiTextureMapMode_Wrap: + prop.texture->sampler->wrapS = SamplerWrap_Repeat; + break; + case aiTextureMapMode_Clamp: + prop.texture->sampler->wrapS = SamplerWrap_Clamp_To_Edge; + break; + case aiTextureMapMode_Mirror: + prop.texture->sampler->wrapS = SamplerWrap_Mirrored_Repeat; + break; + case aiTextureMapMode_Decal: + default: + prop.texture->sampler->wrapS = SamplerWrap_Repeat; + break; + }; + + switch (mapV) { + case aiTextureMapMode_Wrap: + prop.texture->sampler->wrapT = SamplerWrap_Repeat; + break; + case aiTextureMapMode_Clamp: + prop.texture->sampler->wrapT = SamplerWrap_Clamp_To_Edge; + break; + case aiTextureMapMode_Mirror: + prop.texture->sampler->wrapT = SamplerWrap_Mirrored_Repeat; + break; + case aiTextureMapMode_Decal: + default: + prop.texture->sampler->wrapT = SamplerWrap_Repeat; + break; + }; + + // Hard coded Texture filtering options because I do not know where to find them in the aiMaterial. + prop.texture->sampler->magFilter = SamplerMagFilter_Linear; + prop.texture->sampler->minFilter = SamplerMinFilter_Linear; +} + +void glTFExporter::GetMatColorOrTex(const aiMaterial* mat, glTF::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt) +{ + aiString tex; + aiColor4D col; + if (mat->GetTextureCount(tt) > 0) { + if (mat->Get(AI_MATKEY_TEXTURE(tt, 0), tex) == AI_SUCCESS) { + std::string path = tex.C_Str(); + + if (path.size() > 0) { + if (path[0] != '*') { + std::map::iterator it = mTexturesByPath.find(path); + if (it != mTexturesByPath.end()) { + prop.texture = mAsset->textures.Get(it->second); + } + } + + if (!prop.texture) { + std::string texId = mAsset->FindUniqueID("", "texture"); + prop.texture = mAsset->textures.Create(texId); + mTexturesByPath[path] = prop.texture.GetIndex(); + + std::string imgId = mAsset->FindUniqueID("", "image"); + prop.texture->source = mAsset->images.Create(imgId); + + if (path[0] == '*') { // embedded + aiTexture* tex = mScene->mTextures[atoi(&path[1])]; + + uint8_t* data = reinterpret_cast(tex->pcData); + prop.texture->source->SetData(data, tex->mWidth, *mAsset); + + if (tex->achFormatHint[0]) { + std::string mimeType = "image/"; + mimeType += (memcmp(tex->achFormatHint, "jpg", 3) == 0) ? "jpeg" : tex->achFormatHint; + prop.texture->source->mimeType = mimeType; + } + } + else { + prop.texture->source->uri = path; + } + + GetTexSampler(mat, prop); + } + } + } + } + + if (mat->Get(propName, type, idx, col) == AI_SUCCESS) { + prop.color[0] = col.r; prop.color[1] = col.g; prop.color[2] = col.b; prop.color[3] = col.a; + } +} + + +void glTFExporter::ExportMaterials() +{ + aiString aiName; + for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { + const aiMaterial* mat = mScene->mMaterials[i]; + + + std::string name; + if (mat->Get(AI_MATKEY_NAME, aiName) == AI_SUCCESS) { + name = aiName.C_Str(); + } + name = mAsset->FindUniqueID(name, "material"); + + Ref m = mAsset->materials.Create(name); + + GetMatColorOrTex(mat, m->ambient, AI_MATKEY_COLOR_AMBIENT, aiTextureType_AMBIENT); + GetMatColorOrTex(mat, m->diffuse, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_DIFFUSE); + GetMatColorOrTex(mat, m->specular, AI_MATKEY_COLOR_SPECULAR, aiTextureType_SPECULAR); + GetMatColorOrTex(mat, m->emission, AI_MATKEY_COLOR_EMISSIVE, aiTextureType_EMISSIVE); + + m->transparent = mat->Get(AI_MATKEY_OPACITY, m->transparency) == aiReturn_SUCCESS && m->transparency != 1.0; + + GetMatScalar(mat, m->shininess, AI_MATKEY_SHININESS); + } +} + +/* + * Search through node hierarchy and find the node containing the given meshID. + * Returns true on success, and false otherwise. + */ +bool FindMeshNode(Ref& nodeIn, Ref& meshNode, std::string meshID) +{ + for (unsigned int i = 0; i < nodeIn->meshes.size(); ++i) { + if (meshID.compare(nodeIn->meshes[i]->id) == 0) { + meshNode = nodeIn; + return true; + } + } + + for (unsigned int i = 0; i < nodeIn->children.size(); ++i) { + if(FindMeshNode(nodeIn->children[i], meshNode, meshID)) { + return true; + } + } + + return false; +} + +/* + * Find the root joint of the skeleton. + * Starts will any joint node and traces up the tree, + * until a parent is found that does not have a jointName. + * Returns the first parent Ref found that does not have a jointName. + */ +Ref FindSkeletonRootJoint(Ref& skinRef) +{ + Ref startNodeRef; + Ref parentNodeRef; + + // Arbitrarily use the first joint to start the search. + startNodeRef = skinRef->jointNames[0]; + parentNodeRef = skinRef->jointNames[0]; + + do { + startNodeRef = parentNodeRef; + parentNodeRef = startNodeRef->parent; + } while (!parentNodeRef->jointName.empty()); + + return parentNodeRef; +} + +void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref& bufferRef, Ref& skinRef, std::vector& inverseBindMatricesData) +{ + if (aimesh->mNumBones < 1) { + return; + } + + // Store the vertex joint and weight data. + const size_t NumVerts( aimesh->mNumVertices ); + vec4* vertexJointData = new vec4[ NumVerts ]; + vec4* vertexWeightData = new vec4[ NumVerts ]; + int* jointsPerVertex = new int[ NumVerts ]; + for (size_t i = 0; i < NumVerts; ++i) { + jointsPerVertex[i] = 0; + for (size_t j = 0; j < 4; ++j) { + vertexJointData[i][j] = 0; + vertexWeightData[i][j] = 0; + } + } + + for (unsigned int idx_bone = 0; idx_bone < aimesh->mNumBones; ++idx_bone) { + const aiBone* aib = aimesh->mBones[idx_bone]; + + // aib->mName =====> skinRef->jointNames + // Find the node with id = mName. + Ref nodeRef = mAsset.nodes.Get(aib->mName.C_Str()); + nodeRef->jointName = nodeRef->id; + + unsigned int jointNamesIndex; + bool addJointToJointNames = true; + for ( unsigned int idx_joint = 0; idx_joint < skinRef->jointNames.size(); ++idx_joint) { + if (skinRef->jointNames[idx_joint]->jointName.compare(nodeRef->jointName) == 0) { + addJointToJointNames = false; + jointNamesIndex = idx_joint; + } + } + + if (addJointToJointNames) { + skinRef->jointNames.push_back(nodeRef); + + // aib->mOffsetMatrix =====> skinRef->inverseBindMatrices + aiMatrix4x4 tmpMatrix4; + CopyValue(aib->mOffsetMatrix, tmpMatrix4); + inverseBindMatricesData.push_back(tmpMatrix4); + jointNamesIndex = static_cast(inverseBindMatricesData.size() - 1); + } + + // aib->mWeights =====> vertexWeightData + for (unsigned int idx_weights = 0; idx_weights < aib->mNumWeights; ++idx_weights) { + unsigned int vertexId = aib->mWeights[idx_weights].mVertexId; + float vertWeight = aib->mWeights[idx_weights].mWeight; + + // A vertex can only have at most four joint weights. Ignore all others. + if (jointsPerVertex[vertexId] > 3) { + continue; + } + + vertexJointData[vertexId][jointsPerVertex[vertexId]] = jointNamesIndex; + vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; + + jointsPerVertex[vertexId] += 1; + } + + } // End: for-loop mNumMeshes + + Mesh::Primitive& p = meshRef->primitives.back(); + Ref vertexJointAccessor = ExportData(mAsset, skinRef->id, bufferRef, aimesh->mNumVertices, vertexJointData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); + if ( vertexJointAccessor ) { + p.attributes.joint.push_back( vertexJointAccessor ); + } + + Ref vertexWeightAccessor = ExportData(mAsset, skinRef->id, bufferRef, aimesh->mNumVertices, vertexWeightData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); + if ( vertexWeightAccessor ) { + p.attributes.weight.push_back( vertexWeightAccessor ); + } + delete[] jointsPerVertex; + delete[] vertexWeightData; + delete[] vertexJointData; +} + +void glTFExporter::ExportMeshes() +{ + // Not for + // using IndicesType = decltype(aiFace::mNumIndices); + // But yes for + // using IndicesType = unsigned short; + // because "ComponentType_UNSIGNED_SHORT" used for indices. And it's a maximal type according to glTF specification. + typedef unsigned short IndicesType; + + // Variables needed for compression. BEGIN. + // Indices, not pointers - because pointer to buffer is changing while writing to it. + size_t idx_srcdata_begin;// Index of buffer before writing mesh data. Also, index of begin of coordinates array in buffer. + size_t idx_srcdata_normal = SIZE_MAX;// Index of begin of normals array in buffer. SIZE_MAX - mean that mesh has no normals. + std::vector idx_srcdata_tc;// Array of indices. Every index point to begin of texture coordinates array in buffer. + size_t idx_srcdata_ind;// Index of begin of coordinates indices array in buffer. + bool comp_allow;// Point that data of current mesh can be compressed. + // Variables needed for compression. END. + + std::string fname = std::string(mFilename); + std::string bufferIdPrefix = fname.substr(0, fname.rfind(".gltf")); + std::string bufferId = mAsset->FindUniqueID("", bufferIdPrefix.c_str()); + + Ref b = mAsset->GetBodyBuffer(); + if (!b) { + b = mAsset->buffers.Create(bufferId); + } + + //---------------------------------------- + // Initialize variables for the skin + bool createSkin = false; + for (unsigned int idx_mesh = 0; idx_mesh < mScene->mNumMeshes; ++idx_mesh) { + const aiMesh* aim = mScene->mMeshes[idx_mesh]; + if(aim->HasBones()) { + createSkin = true; + break; + } + } + + Ref skinRef; + std::string skinName = mAsset->FindUniqueID("skin", "skin"); + std::vector inverseBindMatricesData; + if(createSkin) { + skinRef = mAsset->skins.Create(skinName); + skinRef->name = skinName; + } + //---------------------------------------- + + for (unsigned int idx_mesh = 0; idx_mesh < mScene->mNumMeshes; ++idx_mesh) { + const aiMesh* aim = mScene->mMeshes[idx_mesh]; + + // Check if compressing requested and mesh can be encoded. +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + comp_allow = mProperties->GetPropertyBool("extensions.Open3DGC.use", false); +#else + comp_allow = false; +#endif + + if(comp_allow && (aim->mPrimitiveTypes == aiPrimitiveType_TRIANGLE) && (aim->mNumVertices > 0) && (aim->mNumFaces > 0)) + { + idx_srcdata_tc.clear(); + idx_srcdata_tc.reserve(AI_MAX_NUMBER_OF_TEXTURECOORDS); + } + else + { + std::string msg; + + if(aim->mPrimitiveTypes != aiPrimitiveType_TRIANGLE) + msg = "all primitives of the mesh must be a triangles."; + else + msg = "mesh must has vertices and faces."; + + DefaultLogger::get()->warn("GLTF: can not use Open3DGC-compression: " + msg); + comp_allow = false; + } + + std::string meshId = mAsset->FindUniqueID(aim->mName.C_Str(), "mesh"); + Ref m = mAsset->meshes.Create(meshId); + m->primitives.resize(1); + Mesh::Primitive& p = m->primitives.back(); + + p.material = mAsset->materials.Get(aim->mMaterialIndex); + + /******************* Vertices ********************/ + // If compression is used then you need parameters of uncompressed region: begin and size. At this step "begin" is stored. + if(comp_allow) idx_srcdata_begin = b->byteLength; + + Ref v = ExportData(*mAsset, meshId, b, aim->mNumVertices, aim->mVertices, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + if (v) p.attributes.position.push_back(v); + + /******************** Normals ********************/ + if(comp_allow && (aim->mNormals != 0)) idx_srcdata_normal = b->byteLength;// Store index of normals array. + + Ref n = ExportData(*mAsset, meshId, b, aim->mNumVertices, aim->mNormals, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + if (n) p.attributes.normal.push_back(n); + + /************** Texture coordinates **************/ + for (int i = 0; i < AI_MAX_NUMBER_OF_TEXTURECOORDS; ++i) { + // Flip UV y coords + if (aim -> mNumUVComponents[i] > 1) { + for (unsigned int j = 0; j < aim->mNumVertices; ++j) { + aim->mTextureCoords[i][j].y = 1 - aim->mTextureCoords[i][j].y; + } + } + + if (aim->mNumUVComponents[i] > 0) { + AttribType::Value type = (aim->mNumUVComponents[i] == 2) ? AttribType::VEC2 : AttribType::VEC3; + + if(comp_allow) idx_srcdata_tc.push_back(b->byteLength);// Store index of texture coordinates array. + + Ref tc = ExportData(*mAsset, meshId, b, aim->mNumVertices, aim->mTextureCoords[i], AttribType::VEC3, type, ComponentType_FLOAT, false); + if (tc) p.attributes.texcoord.push_back(tc); + } + } + + /*************** Vertices indices ****************/ + idx_srcdata_ind = b->byteLength;// Store index of indices array. + + if (aim->mNumFaces > 0) { + std::vector indices; + unsigned int nIndicesPerFace = aim->mFaces[0].mNumIndices; + indices.resize(aim->mNumFaces * nIndicesPerFace); + for (size_t i = 0; i < aim->mNumFaces; ++i) { + for (size_t j = 0; j < nIndicesPerFace; ++j) { + indices[i*nIndicesPerFace + j] = uint16_t(aim->mFaces[i].mIndices[j]); + } + } + + p.indices = ExportData(*mAsset, meshId, b, unsigned(indices.size()), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_SHORT, true); + } + + switch (aim->mPrimitiveTypes) { + case aiPrimitiveType_POLYGON: + p.mode = PrimitiveMode_TRIANGLES; break; // TODO implement this + case aiPrimitiveType_LINE: + p.mode = PrimitiveMode_LINES; break; + case aiPrimitiveType_POINT: + p.mode = PrimitiveMode_POINTS; break; + default: // aiPrimitiveType_TRIANGLE + p.mode = PrimitiveMode_TRIANGLES; + } + + /*************** Skins ****************/ + if(aim->HasBones()) { + ExportSkin(*mAsset, aim, m, b, skinRef, inverseBindMatricesData); + } + + /****************** Compression ******************/ + ///TODO: animation: weights, joints. + if(comp_allow) + { +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + // Only one type of compression supported at now - Open3DGC. + // + o3dgc::BinaryStream bs; + o3dgc::SC3DMCEncoder encoder; + o3dgc::IndexedFaceSet comp_o3dgc_ifs; + o3dgc::SC3DMCEncodeParams comp_o3dgc_params; + + // + // Fill data for encoder. + // + // Quantization + unsigned quant_coord = mProperties->GetPropertyInteger("extensions.Open3DGC.quantization.POSITION", 12); + unsigned quant_normal = mProperties->GetPropertyInteger("extensions.Open3DGC.quantization.NORMAL", 10); + unsigned quant_texcoord = mProperties->GetPropertyInteger("extensions.Open3DGC.quantization.TEXCOORD", 10); + + // Prediction + o3dgc::O3DGCSC3DMCPredictionMode prediction_position = o3dgc::O3DGC_SC3DMC_PARALLELOGRAM_PREDICTION; + o3dgc::O3DGCSC3DMCPredictionMode prediction_normal = o3dgc::O3DGC_SC3DMC_SURF_NORMALS_PREDICTION; + o3dgc::O3DGCSC3DMCPredictionMode prediction_texcoord = o3dgc::O3DGC_SC3DMC_PARALLELOGRAM_PREDICTION; + + // IndexedFacesSet: "Crease angle", "solid", "convex" are set to default. + comp_o3dgc_ifs.SetCCW(true); + comp_o3dgc_ifs.SetIsTriangularMesh(true); + comp_o3dgc_ifs.SetNumFloatAttributes(0); + // Coordinates + comp_o3dgc_params.SetCoordQuantBits(quant_coord); + comp_o3dgc_params.SetCoordPredMode(prediction_position); + comp_o3dgc_ifs.SetNCoord(aim->mNumVertices); + comp_o3dgc_ifs.SetCoord((o3dgc::Real* const)&b->GetPointer()[idx_srcdata_begin]); + // Normals + if(idx_srcdata_normal != SIZE_MAX) + { + comp_o3dgc_params.SetNormalQuantBits(quant_normal); + comp_o3dgc_params.SetNormalPredMode(prediction_normal); + comp_o3dgc_ifs.SetNNormal(aim->mNumVertices); + comp_o3dgc_ifs.SetNormal((o3dgc::Real* const)&b->GetPointer()[idx_srcdata_normal]); + } + + // Texture coordinates + for(size_t num_tc = 0; num_tc < idx_srcdata_tc.size(); num_tc++) + { + size_t num = comp_o3dgc_ifs.GetNumFloatAttributes(); + + comp_o3dgc_params.SetFloatAttributeQuantBits(static_cast(num), quant_texcoord); + comp_o3dgc_params.SetFloatAttributePredMode(static_cast(num), prediction_texcoord); + comp_o3dgc_ifs.SetNFloatAttribute(static_cast(num), aim->mNumVertices);// number of elements. + comp_o3dgc_ifs.SetFloatAttributeDim(static_cast(num), aim->mNumUVComponents[num_tc]);// components per element: aiVector3D => x * float + comp_o3dgc_ifs.SetFloatAttributeType(static_cast(num), o3dgc::O3DGC_IFS_FLOAT_ATTRIBUTE_TYPE_TEXCOORD); + comp_o3dgc_ifs.SetFloatAttribute(static_cast(num), (o3dgc::Real* const)&b->GetPointer()[idx_srcdata_tc[num_tc]]); + comp_o3dgc_ifs.SetNumFloatAttributes(static_cast(num + 1)); + } + + // Coordinates indices + comp_o3dgc_ifs.SetNCoordIndex(aim->mNumFaces); + comp_o3dgc_ifs.SetCoordIndex((IndicesType* const)&b->GetPointer()[idx_srcdata_ind]); + // Prepare to enconding + comp_o3dgc_params.SetNumFloatAttributes(comp_o3dgc_ifs.GetNumFloatAttributes()); + if(mProperties->GetPropertyBool("extensions.Open3DGC.binary", true)) + comp_o3dgc_params.SetStreamType(o3dgc::O3DGC_STREAM_TYPE_BINARY); + else + comp_o3dgc_params.SetStreamType(o3dgc::O3DGC_STREAM_TYPE_ASCII); + + comp_o3dgc_ifs.ComputeMinMax(o3dgc::O3DGC_SC3DMC_MAX_ALL_DIMS); + // + // Encoding + // + encoder.Encode(comp_o3dgc_params, comp_o3dgc_ifs, bs); + // Replace data in buffer. + b->ReplaceData(idx_srcdata_begin, b->byteLength - idx_srcdata_begin, bs.GetBuffer(), bs.GetSize()); + // + // Add information about extension to mesh. + // + // Create extension structure. + Mesh::SCompression_Open3DGC* ext = new Mesh::SCompression_Open3DGC; + + // Fill it. + ext->Buffer = b->id; + ext->Offset = idx_srcdata_begin; + ext->Count = b->byteLength - idx_srcdata_begin; + ext->Binary = mProperties->GetPropertyBool("extensions.Open3DGC.binary"); + ext->IndicesCount = comp_o3dgc_ifs.GetNCoordIndex() * 3; + ext->VerticesCount = comp_o3dgc_ifs.GetNCoord(); + // And assign to mesh. + m->Extension.push_back(ext); +#endif + }// if(comp_allow) + }// for (unsigned int i = 0; i < mScene->mNumMeshes; ++i) + + //---------------------------------------- + // Finish the skin + // Create the Accessor for skinRef->inverseBindMatrices + if (createSkin) { + mat4* invBindMatrixData = new mat4[inverseBindMatricesData.size()]; + for ( unsigned int idx_joint = 0; idx_joint < inverseBindMatricesData.size(); ++idx_joint) { + CopyValue(inverseBindMatricesData[idx_joint], invBindMatrixData[idx_joint]); + } + + Ref invBindMatrixAccessor = ExportData(*mAsset, skinName, b, static_cast(inverseBindMatricesData.size()), invBindMatrixData, AttribType::MAT4, AttribType::MAT4, ComponentType_FLOAT); + if (invBindMatrixAccessor) skinRef->inverseBindMatrices = invBindMatrixAccessor; + + // Identity Matrix =====> skinRef->bindShapeMatrix + // Temporary. Hard-coded identity matrix here + skinRef->bindShapeMatrix.isPresent = true; + IdentityMatrix4(skinRef->bindShapeMatrix.value); + + // Find node that contains this mesh and add "skeletons" and "skin" attributes to that node. + Ref rootNode = mAsset->nodes.Get(unsigned(0)); + Ref meshNode; + std::string meshID = mAsset->meshes.Get(unsigned(0))->id; + FindMeshNode(rootNode, meshNode, meshID); + + Ref rootJoint = FindSkeletonRootJoint(skinRef); + meshNode->skeletons.push_back(rootJoint); + meshNode->skin = skinRef; + } +} + +/* + * Export the root node of the node hierarchy. + * Calls ExportNode for all children. + */ +unsigned int glTFExporter::ExportNodeHierarchy(const aiNode* n) +{ + Ref node = mAsset->nodes.Create(mAsset->FindUniqueID(n->mName.C_Str(), "node")); + + if (!n->mTransformation.IsIdentity()) { + node->matrix.isPresent = true; + CopyValue(n->mTransformation, node->matrix.value); + } + + for (unsigned int i = 0; i < n->mNumMeshes; ++i) { + node->meshes.push_back(mAsset->meshes.Get(n->mMeshes[i])); + } + + for (unsigned int i = 0; i < n->mNumChildren; ++i) { + unsigned int idx = ExportNode(n->mChildren[i], node); + node->children.push_back(mAsset->nodes.Get(idx)); + } + + return node.GetIndex(); +} + +/* + * Export node and recursively calls ExportNode for all children. + * Since these nodes are not the root node, we also export the parent Ref + */ +unsigned int glTFExporter::ExportNode(const aiNode* n, Ref& parent) +{ + Ref node = mAsset->nodes.Create(mAsset->FindUniqueID(n->mName.C_Str(), "node")); + + node->parent = parent; + + if (!n->mTransformation.IsIdentity()) { + node->matrix.isPresent = true; + CopyValue(n->mTransformation, node->matrix.value); + } + + for (unsigned int i = 0; i < n->mNumMeshes; ++i) { + node->meshes.push_back(mAsset->meshes.Get(n->mMeshes[i])); + } + + for (unsigned int i = 0; i < n->mNumChildren; ++i) { + unsigned int idx = ExportNode(n->mChildren[i], node); + node->children.push_back(mAsset->nodes.Get(idx)); + } + + return node.GetIndex(); +} + + +void glTFExporter::ExportScene() +{ + const char* sceneName = "defaultScene"; + Ref scene = mAsset->scenes.Create(sceneName); + + // root node will be the first one exported (idx 0) + if (mAsset->nodes.Size() > 0) { + scene->nodes.push_back(mAsset->nodes.Get(0u)); + } + + // set as the default scene + mAsset->scene = scene; +} + +void glTFExporter::ExportMetadata() +{ + glTF::AssetMetadata& asset = mAsset->asset; + asset.version = 1; + + char buffer[256]; + ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", + aiGetVersionMajor(), aiGetVersionMinor(), aiGetVersionRevision()); + + asset.generator = buffer; +} + +inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref& animRef, Ref& buffer, const aiNodeAnim* nodeChannel, float ticksPerSecond) +{ + // Loop over the data and check to see if it exactly matches an existing buffer. + // If yes, then reference the existing corresponding accessor. + // Otherwise, add to the buffer and create a new accessor. + + size_t counts[3] = { + nodeChannel->mNumPositionKeys, + nodeChannel->mNumScalingKeys, + nodeChannel->mNumRotationKeys, + }; + size_t numKeyframes = 1; + for (int i = 0; i < 3; ++i) { + if (counts[i] > numKeyframes) { + numKeyframes = counts[i]; + } + } + + //------------------------------------------------------- + // Extract TIME parameter data. + // Check if the timeStamps are the same for mPositionKeys, mRotationKeys, and mScalingKeys. + if(nodeChannel->mNumPositionKeys > 0) { + typedef float TimeType; + std::vector timeData; + timeData.resize(numKeyframes); + for (size_t i = 0; i < numKeyframes; ++i) { + size_t frameIndex = i * nodeChannel->mNumPositionKeys / numKeyframes; + // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. + // Check if we have to cast type here. e.g. uint16_t() + timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; + } + + Ref timeAccessor = ExportData(mAsset, animId, buffer, numKeyframes, &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); + if (timeAccessor) animRef->Parameters.TIME = timeAccessor; + } + + //------------------------------------------------------- + // Extract translation parameter data + if(nodeChannel->mNumPositionKeys > 0) { + C_STRUCT aiVector3D* translationData = new aiVector3D[numKeyframes]; + for (size_t i = 0; i < numKeyframes; ++i) { + size_t frameIndex = i * nodeChannel->mNumPositionKeys / numKeyframes; + translationData[i] = nodeChannel->mPositionKeys[frameIndex].mValue; + } + + Ref tranAccessor = ExportData(mAsset, animId, buffer, numKeyframes, translationData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + if ( tranAccessor ) { + animRef->Parameters.translation = tranAccessor; + } + delete[] translationData; + } + + //------------------------------------------------------- + // Extract scale parameter data + if(nodeChannel->mNumScalingKeys > 0) { + C_STRUCT aiVector3D* scaleData = new aiVector3D[numKeyframes]; + for (size_t i = 0; i < numKeyframes; ++i) { + size_t frameIndex = i * nodeChannel->mNumScalingKeys / numKeyframes; + scaleData[i] = nodeChannel->mScalingKeys[frameIndex].mValue; + } + + Ref scaleAccessor = ExportData(mAsset, animId, buffer, numKeyframes, scaleData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + if ( scaleAccessor ) { + animRef->Parameters.scale = scaleAccessor; + } + delete[] scaleData; + } + + //------------------------------------------------------- + // Extract rotation parameter data + if(nodeChannel->mNumRotationKeys > 0) { + vec4* rotationData = new vec4[numKeyframes]; + for (size_t i = 0; i < numKeyframes; ++i) { + size_t frameIndex = i * nodeChannel->mNumRotationKeys / numKeyframes; + rotationData[i][0] = nodeChannel->mRotationKeys[frameIndex].mValue.x; + rotationData[i][1] = nodeChannel->mRotationKeys[frameIndex].mValue.y; + rotationData[i][2] = nodeChannel->mRotationKeys[frameIndex].mValue.z; + rotationData[i][3] = nodeChannel->mRotationKeys[frameIndex].mValue.w; + } + + Ref rotAccessor = ExportData(mAsset, animId, buffer, numKeyframes, rotationData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); + if ( rotAccessor ) { + animRef->Parameters.rotation = rotAccessor; + } + delete[] rotationData; + } +} + +void glTFExporter::ExportAnimations() +{ + Ref bufferRef = mAsset->buffers.Get(unsigned (0)); + + for (unsigned int i = 0; i < mScene->mNumAnimations; ++i) { + const aiAnimation* anim = mScene->mAnimations[i]; + + std::string nameAnim = "anim"; + if (anim->mName.length > 0) { + nameAnim = anim->mName.C_Str(); + } + + for (unsigned int channelIndex = 0; channelIndex < anim->mNumChannels; ++channelIndex) { + const aiNodeAnim* nodeChannel = anim->mChannels[channelIndex]; + + // It appears that assimp stores this type of animation as multiple animations. + // where each aiNodeAnim in mChannels animates a specific node. + std::string name = nameAnim + "_" + to_string(channelIndex); + name = mAsset->FindUniqueID(name, "animation"); + Ref animRef = mAsset->animations.Create(name); + + /******************* Parameters ********************/ + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); + + for (unsigned int j = 0; j < 3; ++j) { + std::string channelType; + int channelSize; + switch (j) { + case 0: + channelType = "rotation"; + channelSize = nodeChannel->mNumRotationKeys; + break; + case 1: + channelType = "scale"; + channelSize = nodeChannel->mNumScalingKeys; + break; + case 2: + channelType = "translation"; + channelSize = nodeChannel->mNumPositionKeys; + break; + } + + if (channelSize < 1) { continue; } + + Animation::AnimChannel tmpAnimChannel; + Animation::AnimSampler tmpAnimSampler; + + tmpAnimChannel.sampler = name + "_" + channelType; + tmpAnimChannel.target.path = channelType; + tmpAnimSampler.output = channelType; + tmpAnimSampler.id = name + "_" + channelType; + + tmpAnimChannel.target.id = mAsset->nodes.Get(nodeChannel->mNodeName.C_Str()); + + tmpAnimSampler.input = "TIME"; + tmpAnimSampler.interpolation = "LINEAR"; + + animRef->Channels.push_back(tmpAnimChannel); + animRef->Samplers.push_back(tmpAnimSampler); + } + + } + + // Assimp documentation staes this is not used (not implemented) + // for (unsigned int channelIndex = 0; channelIndex < anim->mNumMeshChannels; ++channelIndex) { + // const aiMeshAnim* meshChannel = anim->mMeshChannels[channelIndex]; + // } + + } // End: for-loop mNumAnimations +} + + +#endif // ASSIMP_BUILD_NO_GLTF_EXPORTER +#endif // ASSIMP_BUILD_NO_EXPORT diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h new file mode 100644 index 000000000..c813fff44 --- /dev/null +++ b/code/glTF2Exporter.h @@ -0,0 +1,118 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +/** @file GltfExporter.h +* Declares the exporter class to write a scene to a gltf/glb file +*/ +#ifndef AI_GLTFEXPORTER_H_INC +#define AI_GLTFEXPORTER_H_INC + +#ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER + +#include +#include + +#include +#include +#include +#include + +struct aiScene; +struct aiNode; +struct aiMaterial; + +namespace glTF +{ + template + class Ref; + + class Asset; + struct TexProperty; + struct Node; +} + +namespace Assimp +{ + class IOSystem; + class IOStream; + class ExportProperties; + + // ------------------------------------------------------------------------------------------------ + /** Helper class to export a given scene to an glTF file. */ + // ------------------------------------------------------------------------------------------------ + class glTFExporter + { + public: + /// Constructor for a specific scene to export + glTFExporter(const char* filename, IOSystem* pIOSystem, const aiScene* pScene, + const ExportProperties* pProperties, bool binary); + + private: + + const char* mFilename; + IOSystem* mIOSystem; + const aiScene* mScene; + const ExportProperties* mProperties; + + std::map mTexturesByPath; + + std::shared_ptr mAsset; + + std::vector mBodyData; + + void WriteBinaryData(IOStream* outfile, std::size_t sceneLength); + + void GetTexSampler(const aiMaterial* mat, glTF::TexProperty& prop); + void GetMatColorOrTex(const aiMaterial* mat, glTF::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt); + void ExportMetadata(); + void ExportMaterials(); + void ExportMeshes(); + unsigned int ExportNodeHierarchy(const aiNode* n); + unsigned int ExportNode(const aiNode* node, glTF::Ref& parent); + void ExportScene(); + void ExportAnimations(); + }; + +} + +#endif // ASSIMP_BUILD_NO_GLTF_IMPORTER + +#endif // AI_GLTFEXPORTER_H_INC From d7cbbaf23e6a0a066c4869d1a211850c4fc59c05 Mon Sep 17 00:00:00 2001 From: jamesgk Date: Thu, 20 Jul 2017 10:59:21 -0700 Subject: [PATCH 022/305] Compile with glTF2 export option (currently same as glTF1 output) --- code/CMakeLists.txt | 6 +++++ code/Exporter.cpp | 3 +++ code/glTF2Asset.h | 10 ++++---- code/glTF2Asset.inl | 2 +- code/glTF2AssetWriter.h | 12 +++++----- code/glTF2AssetWriter.inl | 2 +- code/glTF2Exporter.cpp | 48 ++++++++++++++++----------------------- code/glTF2Exporter.h | 20 ++++++++-------- 8 files changed, 52 insertions(+), 51 deletions(-) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 0477f5fef..8c9a248c4 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -661,6 +661,12 @@ ADD_ASSIMP_IMPORTER( GLTF glTFImporter.h glTFExporter.h glTFExporter.cpp + glTF2Asset.h + glTF2Asset.inl + glTF2AssetWriter.h + glTF2AssetWriter.inl + glTF2Exporter.h + glTF2Exporter.cpp ) ADD_ASSIMP_IMPORTER( 3MF diff --git a/code/Exporter.cpp b/code/Exporter.cpp index f03702869..7af7d2cf5 100644 --- a/code/Exporter.cpp +++ b/code/Exporter.cpp @@ -90,6 +90,7 @@ void ExportScenePlyBinary(const char*, IOSystem*, const aiScene*, const ExportPr void ExportScene3DS(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneGLTF(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneGLB(const char*, IOSystem*, const aiScene*, const ExportProperties*); +void ExportSceneGLTF2(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneAssbin(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneAssxml(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneX3D(const char*, IOSystem*, const aiScene*, const ExportProperties*); @@ -144,6 +145,8 @@ Exporter::ExportFormatEntry gExporters[] = aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_SortByPType), Exporter::ExportFormatEntry( "glb", "GL Transmission Format (binary)", "glb", &ExportSceneGLB, aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_SortByPType), + Exporter::ExportFormatEntry( "gltf2", "GL Transmission Format v. 2", "gltf", &ExportSceneGLTF2, + aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_SortByPType), #endif #ifndef ASSIMP_BUILD_NO_ASSBIN_EXPORTER diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 41d0dfd06..3331c1128 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -46,8 +46,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * KHR_binary_glTF: full * KHR_materials_common: full */ -#ifndef GLTFASSET_H_INC -#define GLTFASSET_H_INC +#ifndef GLTF2ASSET_H_INC +#define GLTF2ASSET_H_INC #ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER @@ -89,7 +89,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # endif #endif -namespace glTF +namespace glTF2 { #ifdef ASSIMP_API using Assimp::IOStream; @@ -1187,8 +1187,8 @@ namespace glTF } // Include the implementation of the methods -#include "glTFAsset.inl" +#include "glTF2Asset.inl" #endif // ASSIMP_BUILD_NO_GLTF_IMPORTER -#endif // GLTFASSET_H_INC +#endif // GLTF2ASSET_H_INC diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index fe29dde70..9065cedb5 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -51,7 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace Assimp; -namespace glTF { +namespace glTF2 { namespace { diff --git a/code/glTF2AssetWriter.h b/code/glTF2AssetWriter.h index 186d32a15..976f23f9a 100644 --- a/code/glTF2AssetWriter.h +++ b/code/glTF2AssetWriter.h @@ -46,14 +46,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * KHR_binary_glTF: full * KHR_materials_common: full */ -#ifndef GLTFASSETWRITER_H_INC -#define GLTFASSETWRITER_H_INC +#ifndef GLTF2ASSETWRITER_H_INC +#define GLTF2ASSETWRITER_H_INC #ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER -#include "glTFAsset.h" +#include "glTF2Asset.h" -namespace glTF +namespace glTF2 { using rapidjson::MemoryPoolAllocator; @@ -88,8 +88,8 @@ public: } // Include the implementation of the methods -#include "glTFAssetWriter.inl" +#include "glTF2AssetWriter.inl" #endif // ASSIMP_BUILD_NO_GLTF_IMPORTER -#endif // GLTFASSETWRITER_H_INC +#endif // GLTF2ASSETWRITER_H_INC diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index d3a553c27..9bf1c5697 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -namespace glTF { +namespace glTF2 { using rapidjson::StringBuffer; using rapidjson::PrettyWriter; diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index aeb5069aa..e6dc2c4b1 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_EXPORT #ifndef ASSIMP_BUILD_NO_GLTF_EXPORTER -#include "glTFExporter.h" +#include "glTF2Exporter.h" #include "Exceptional.h" #include "StringComparison.h" @@ -60,7 +60,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#include "glTFAssetWriter.h" +#include "glTF2AssetWriter.h" #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC // Header files, Open3DGC. @@ -70,29 +70,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace rapidjson; using namespace Assimp; -using namespace glTF; +using namespace glTF2; namespace Assimp { // ------------------------------------------------------------------------------------------------ // Worker function for exporting a scene to GLTF. Prototyped and registered in Exporter.cpp - void ExportSceneGLTF(const char* pFile, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties) + void ExportSceneGLTF2(const char* pFile, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties) { // invoke the exporter - glTFExporter exporter(pFile, pIOSystem, pScene, pProperties, false); - } - - // ------------------------------------------------------------------------------------------------ - // Worker function for exporting a scene to GLB. Prototyped and registered in Exporter.cpp - void ExportSceneGLB(const char* pFile, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties) - { - // invoke the exporter - glTFExporter exporter(pFile, pIOSystem, pScene, pProperties, true); + glTF2Exporter exporter(pFile, pIOSystem, pScene, pProperties, false); } } // end of namespace Assimp -glTFExporter::glTFExporter(const char* filename, IOSystem* pIOSystem, const aiScene* pScene, +glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties, bool isBinary) : mFilename(filename) , mIOSystem(pIOSystem) @@ -112,7 +104,7 @@ glTFExporter::glTFExporter(const char* filename, IOSystem* pIOSystem, const aiSc mScene = sceneCopy.get(); - mAsset.reset( new glTF::Asset( pIOSystem ) ); + mAsset.reset( new Asset( pIOSystem ) ); if (isBinary) { mAsset->SetAsBinary(); @@ -138,7 +130,7 @@ glTFExporter::glTFExporter(const char* filename, IOSystem* pIOSystem, const aiSc ExportAnimations(); - glTF::AssetWriter writer(*mAsset); + AssetWriter writer(*mAsset); if (isBinary) { writer.WriteGLBFile(filename); @@ -151,7 +143,7 @@ glTFExporter::glTFExporter(const char* filename, IOSystem* pIOSystem, const aiSc * Copy a 4x4 matrix from struct aiMatrix to typedef mat4. * Also converts from row-major to column-major storage. */ -static void CopyValue(const aiMatrix4x4& v, glTF::mat4& o) +static void CopyValue(const aiMatrix4x4& v, mat4& o) { o[ 0] = v.a1; o[ 1] = v.b1; o[ 2] = v.c1; o[ 3] = v.d1; o[ 4] = v.a2; o[ 5] = v.b2; o[ 6] = v.c2; o[ 7] = v.d2; @@ -167,7 +159,7 @@ static void CopyValue(const aiMatrix4x4& v, aiMatrix4x4& o) o.d1 = v.d1; o.d2 = v.d2; o.d3 = v.d3; o.d4 = v.d4; } -static void IdentityMatrix4(glTF::mat4& o) +static void IdentityMatrix4(mat4& o) { o[ 0] = 1; o[ 1] = 0; o[ 2] = 0; o[ 3] = 0; o[ 4] = 0; o[ 5] = 1; o[ 6] = 0; o[ 7] = 0; @@ -248,7 +240,7 @@ namespace { } } -void glTFExporter::GetTexSampler(const aiMaterial* mat, glTF::TexProperty& prop) +void glTF2Exporter::GetTexSampler(const aiMaterial* mat, TexProperty& prop) { std::string samplerId = mAsset->FindUniqueID("", "sampler"); prop.texture->sampler = mAsset->samplers.Create(samplerId); @@ -294,7 +286,7 @@ void glTFExporter::GetTexSampler(const aiMaterial* mat, glTF::TexProperty& prop) prop.texture->sampler->minFilter = SamplerMinFilter_Linear; } -void glTFExporter::GetMatColorOrTex(const aiMaterial* mat, glTF::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt) +void glTF2Exporter::GetMatColorOrTex(const aiMaterial* mat, TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt) { aiString tex; aiColor4D col; @@ -346,7 +338,7 @@ void glTFExporter::GetMatColorOrTex(const aiMaterial* mat, glTF::TexProperty& pr } -void glTFExporter::ExportMaterials() +void glTF2Exporter::ExportMaterials() { aiString aiName; for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { @@ -496,7 +488,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref node = mAsset->nodes.Create(mAsset->FindUniqueID(n->mName.C_Str(), "node")); @@ -793,7 +785,7 @@ unsigned int glTFExporter::ExportNodeHierarchy(const aiNode* n) * Export node and recursively calls ExportNode for all children. * Since these nodes are not the root node, we also export the parent Ref */ -unsigned int glTFExporter::ExportNode(const aiNode* n, Ref& parent) +unsigned int glTF2Exporter::ExportNode(const aiNode* n, Ref& parent) { Ref node = mAsset->nodes.Create(mAsset->FindUniqueID(n->mName.C_Str(), "node")); @@ -817,7 +809,7 @@ unsigned int glTFExporter::ExportNode(const aiNode* n, Ref& parent) } -void glTFExporter::ExportScene() +void glTF2Exporter::ExportScene() { const char* sceneName = "defaultScene"; Ref scene = mAsset->scenes.Create(sceneName); @@ -831,9 +823,9 @@ void glTFExporter::ExportScene() mAsset->scene = scene; } -void glTFExporter::ExportMetadata() +void glTF2Exporter::ExportMetadata() { - glTF::AssetMetadata& asset = mAsset->asset; + AssetMetadata& asset = mAsset->asset; asset.version = 1; char buffer[256]; @@ -931,7 +923,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref bufferRef = mAsset->buffers.Get(unsigned (0)); diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index c813fff44..1378fa9f8 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -42,8 +42,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file GltfExporter.h * Declares the exporter class to write a scene to a gltf/glb file */ -#ifndef AI_GLTFEXPORTER_H_INC -#define AI_GLTFEXPORTER_H_INC +#ifndef AI_GLTF2EXPORTER_H_INC +#define AI_GLTF2EXPORTER_H_INC #ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER @@ -59,7 +59,7 @@ struct aiScene; struct aiNode; struct aiMaterial; -namespace glTF +namespace glTF2 { template class Ref; @@ -78,11 +78,11 @@ namespace Assimp // ------------------------------------------------------------------------------------------------ /** Helper class to export a given scene to an glTF file. */ // ------------------------------------------------------------------------------------------------ - class glTFExporter + class glTF2Exporter { public: /// Constructor for a specific scene to export - glTFExporter(const char* filename, IOSystem* pIOSystem, const aiScene* pScene, + glTF2Exporter(const char* filename, IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties, bool binary); private: @@ -94,19 +94,19 @@ namespace Assimp std::map mTexturesByPath; - std::shared_ptr mAsset; + std::shared_ptr mAsset; std::vector mBodyData; void WriteBinaryData(IOStream* outfile, std::size_t sceneLength); - void GetTexSampler(const aiMaterial* mat, glTF::TexProperty& prop); - void GetMatColorOrTex(const aiMaterial* mat, glTF::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt); + void GetTexSampler(const aiMaterial* mat, glTF2::TexProperty& prop); + void GetMatColorOrTex(const aiMaterial* mat, glTF2::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt); void ExportMetadata(); void ExportMaterials(); void ExportMeshes(); unsigned int ExportNodeHierarchy(const aiNode* n); - unsigned int ExportNode(const aiNode* node, glTF::Ref& parent); + unsigned int ExportNode(const aiNode* node, glTF2::Ref& parent); void ExportScene(); void ExportAnimations(); }; @@ -115,4 +115,4 @@ namespace Assimp #endif // ASSIMP_BUILD_NO_GLTF_IMPORTER -#endif // AI_GLTFEXPORTER_H_INC +#endif // AI_GLTF2EXPORTER_H_INC From bb55246c180f4722d2dda5913199e66cd14a8f03 Mon Sep 17 00:00:00 2001 From: jamesgk Date: Tue, 25 Jul 2017 17:26:18 -0700 Subject: [PATCH 023/305] Export glTF 2 --- code/glTF2Asset.h | 23 ++++++-- code/glTF2Asset.inl | 1 + code/glTF2AssetWriter.inl | 112 ++++++++++++++++---------------------- code/glTF2Exporter.cpp | 38 +++++++++---- 4 files changed, 93 insertions(+), 81 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 3331c1128..36caba2cb 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -177,7 +177,7 @@ namespace glTF2 struct GLB_Header { uint8_t magic[4]; //!< Magic number: "glTF" - uint32_t version; //!< Version number (always 1 as of the last update) + uint32_t version; //!< Version number uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes uint32_t sceneLength; //!< Length, in bytes, of the glTF scene uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) @@ -381,6 +381,7 @@ namespace glTF2 //! Base classe for all glTF top-level objects struct Object { + int index; //!< The index of this object within its property container std::string id; //!< The globally unique ID used to reference this object std::string name; //!< The user-defined name of this object @@ -952,10 +953,10 @@ namespace glTF2 }; struct AnimChannel { - std::string sampler; //!< The ID of one sampler present in the containing animation's samplers property. + int sampler; //!< The index of a sampler in the containing animation's samplers property. struct AnimTarget { - Ref id; //!< The ID of the node to animate. + Ref node; //!< The node to animate. std::string path; //!< The name of property of the node to animate ("translation", "rotation", or "scale"). } target; }; @@ -977,6 +978,20 @@ namespace glTF2 Animation() {} void Read(Value& obj, Asset& r); + + //! Get accessor given an animation parameter name. + Ref GetAccessor(std::string name) { + if (name == "TIME") { + return Parameters.TIME; + } else if (name == "rotation") { + return Parameters.rotation; + } else if (name == "scale") { + return Parameters.scale; + } else if (name == "translation") { + return Parameters.translation; + } + return Ref(); + } }; @@ -1058,7 +1073,7 @@ namespace glTF2 std::string version; //!< Specifies the target rendering API (default: "1.0.3") } profile; //!< Specifies the target rendering API and version, e.g., WebGL 1.0.3. (default: {}) - int version; //!< The glTF format version (should be 1) + int version; //!< The glTF format version void Read(Document& doc); diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 9065cedb5..33598e243 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -242,6 +242,7 @@ Ref LazyDict::Create(const char* id) } T* inst = new T(); inst->id = id; + inst->index = mObjs.size(); return Add(inst); } diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 9bf1c5697..2c3f2fffd 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -79,7 +79,7 @@ namespace glTF2 { lst.SetArray(); lst.Reserve(unsigned(v.size()), al); for (size_t i = 0; i < v.size(); ++i) { - lst.PushBack(StringRef(v[i]->id), al); + lst.PushBack(v[i]->index, al); } obj.AddMember(StringRef(fieldId), lst, al); } @@ -89,7 +89,7 @@ namespace glTF2 { inline void Write(Value& obj, Accessor& a, AssetWriter& w) { - obj.AddMember("bufferView", Value(a.bufferView->id, w.mAl).Move(), w.mAl); + obj.AddMember("bufferView", a.bufferView->index, w.mAl); obj.AddMember("byteOffset", a.byteOffset, w.mAl); obj.AddMember("byteStride", a.byteStride, w.mAl); obj.AddMember("componentType", int(a.componentType), w.mAl); @@ -113,12 +113,13 @@ namespace glTF2 { Value valChannel; valChannel.SetObject(); { - valChannel.AddMember("sampler", c.sampler, w.mAl); + Animation::AnimSampler& s = a.Samplers[c.sampler]; + valChannel.AddMember("sampler", s.id, w.mAl); Value valTarget; valTarget.SetObject(); { - valTarget.AddMember("id", StringRef(c.target.id->id), w.mAl); + valTarget.AddMember("id", StringRef(c.target.node->id), w.mAl); valTarget.AddMember("path", c.target.path, w.mAl); } valChannel.AddMember("target", valTarget, w.mAl); @@ -127,61 +128,35 @@ namespace glTF2 { } obj.AddMember("channels", channels, w.mAl); - /****************** Parameters *******************/ - Value valParameters; - valParameters.SetObject(); - { - if (a.Parameters.TIME) { - valParameters.AddMember("TIME", StringRef(a.Parameters.TIME->id), w.mAl); - } - if (a.Parameters.rotation) { - valParameters.AddMember("rotation", StringRef(a.Parameters.rotation->id), w.mAl); - } - if (a.Parameters.scale) { - valParameters.AddMember("scale", StringRef(a.Parameters.scale->id), w.mAl); - } - if (a.Parameters.translation) { - valParameters.AddMember("translation", StringRef(a.Parameters.translation->id), w.mAl); - } - } - obj.AddMember("parameters", valParameters, w.mAl); - /****************** Samplers *******************/ Value valSamplers; - valSamplers.SetObject(); + valSamplers.SetArray(); for (size_t i = 0; i < unsigned(a.Samplers.size()); ++i) { Animation::AnimSampler& s = a.Samplers[i]; Value valSampler; valSampler.SetObject(); { - valSampler.AddMember("input", s.input, w.mAl); + Ref inputAccessor = a.GetAccessor(s.input); + Ref outputAccessor = a.GetAccessor(s.output); + valSampler.AddMember("input", inputAccessor->index, w.mAl); valSampler.AddMember("interpolation", s.interpolation, w.mAl); - valSampler.AddMember("output", s.output, w.mAl); + valSampler.AddMember("output", outputAccessor->index, w.mAl); } - valSamplers.AddMember(StringRef(s.id), valSampler, w.mAl); + valSamplers.PushBack(valSampler, w.mAl); } obj.AddMember("samplers", valSamplers, w.mAl); } inline void Write(Value& obj, Buffer& b, AssetWriter& w) { - const char* type; - switch (b.type) { - case Buffer::Type_text: - type = "text"; break; - default: - type = "arraybuffer"; - } - obj.AddMember("byteLength", static_cast(b.byteLength), w.mAl); - obj.AddMember("type", StringRef(type), w.mAl); obj.AddMember("uri", Value(b.GetURI(), w.mAl).Move(), w.mAl); } inline void Write(Value& obj, BufferView& bv, AssetWriter& w) { - obj.AddMember("buffer", Value(bv.buffer->id, w.mAl).Move(), w.mAl); + obj.AddMember("buffer", bv.buffer->index, w.mAl); obj.AddMember("byteOffset", static_cast(bv.byteOffset), w.mAl); obj.AddMember("byteLength", static_cast(bv.byteLength), w.mAl); obj.AddMember("target", int(bv.target), w.mAl); @@ -200,7 +175,7 @@ namespace glTF2 { exts.SetObject(); ext.SetObject(); - ext.AddMember("bufferView", StringRef(img.bufferView->id), w.mAl); + ext.AddMember("bufferView", img.bufferView->index, w.mAl); if (!img.mimeType.empty()) ext.AddMember("mimeType", StringRef(img.mimeType), w.mAl); @@ -224,9 +199,12 @@ namespace glTF2 { namespace { inline void WriteColorOrTex(Value& obj, TexProperty& prop, const char* propName, MemoryPoolAllocator<>& al) { - if (prop.texture) - obj.AddMember(StringRef(propName), Value(prop.texture->id, al).Move(), al); - else { + if (prop.texture) { + Value tex; + tex.SetObject(); + tex.AddMember("index", prop.texture->index, al); + obj.AddMember(StringRef(propName), tex, al); + } else { Value col; obj.AddMember(StringRef(propName), MakeValue(col, prop.color, al), al); } @@ -238,17 +216,21 @@ namespace glTF2 { Value v; v.SetObject(); { - WriteColorOrTex(v, m.ambient, "ambient", w.mAl); - WriteColorOrTex(v, m.diffuse, "diffuse", w.mAl); - WriteColorOrTex(v, m.specular, "specular", w.mAl); - WriteColorOrTex(v, m.emission, "emission", w.mAl); + WriteColorOrTex(v, m.ambient, m.ambient.texture ? "ambientTexture" : "ambientFactor", w.mAl); + WriteColorOrTex(v, m.diffuse, m.diffuse.texture ? "diffuseTexture" : "diffuseFactor", w.mAl); + WriteColorOrTex(v, m.specular, m.specular.texture ? "specularTexture" : "specularFactor", w.mAl); + WriteColorOrTex(v, m.emission, m.emission.texture ? "emissionTexture" : "emissionFactor", w.mAl); if (m.transparent) v.AddMember("transparency", m.transparency, w.mAl); - v.AddMember("shininess", m.shininess, w.mAl); + v.AddMember("shininessFactor", m.shininess, w.mAl); } - obj.AddMember("values", v, w.mAl); + v.AddMember("type", "commonPhong", w.mAl); + Value ext; + ext.SetObject(); + ext.AddMember("KHR_materials_common", v, w.mAl); + obj.AddMember("extensions", ext, w.mAl); } namespace { @@ -257,13 +239,13 @@ namespace glTF2 { { if (lst.empty()) return; if (lst.size() == 1 && !forceNumber) { - attrs.AddMember(StringRef(semantic), Value(lst[0]->id, w.mAl).Move(), w.mAl); + attrs.AddMember(StringRef(semantic), lst[0]->index, w.mAl); } else { for (size_t i = 0; i < lst.size(); ++i) { char buffer[32]; ai_snprintf(buffer, 32, "%s_%d", semantic, int(i)); - attrs.AddMember(Value(buffer, w.mAl).Move(), Value(lst[i]->id, w.mAl).Move(), w.mAl); + attrs.AddMember(Value(buffer, w.mAl).Move(), lst[i]->index, w.mAl); } } } @@ -337,10 +319,10 @@ namespace glTF2 { prim.AddMember("mode", Value(int(p.mode)).Move(), w.mAl); if (p.material) - prim.AddMember("material", p.material->id, w.mAl); + prim.AddMember("material", p.material->index, w.mAl); if (p.indices) - prim.AddMember("indices", Value(p.indices->id, w.mAl).Move(), w.mAl); + prim.AddMember("indices", p.indices->index, w.mAl); Value attrs; attrs.SetObject(); @@ -390,7 +372,7 @@ namespace glTF2 { AddRefsVector(obj, "skeletons", n.skeletons, w.mAl); if (n.skin) { - obj.AddMember("skin", Value(n.skin->id, w.mAl).Move(), w.mAl); + obj.AddMember("skin", n.skin->index, w.mAl); } if (!n.jointName.empty()) { @@ -437,9 +419,9 @@ namespace glTF2 { vJointNames.Reserve(unsigned(b.jointNames.size()), w.mAl); for (size_t i = 0; i < unsigned(b.jointNames.size()); ++i) { - vJointNames.PushBack(StringRef(b.jointNames[i]->jointName), w.mAl); + vJointNames.PushBack(b.jointNames[i]->index, w.mAl); } - obj.AddMember("jointNames", vJointNames, w.mAl); + obj.AddMember("joints", vJointNames, w.mAl); if (b.bindShapeMatrix.isPresent) { Value val; @@ -447,7 +429,7 @@ namespace glTF2 { } if (b.inverseBindMatrices) { - obj.AddMember("inverseBindMatrices", Value(b.inverseBindMatrices->id, w.mAl).Move(), w.mAl); + obj.AddMember("inverseBindMatrices", b.inverseBindMatrices->index, w.mAl); } } @@ -460,10 +442,10 @@ namespace glTF2 { inline void Write(Value& obj, Texture& tex, AssetWriter& w) { if (tex.source) { - obj.AddMember("source", Value(tex.source->id, w.mAl).Move(), w.mAl); + obj.AddMember("source", tex.source->index, w.mAl); } if (tex.sampler) { - obj.AddMember("sampler", Value(tex.sampler->id, w.mAl).Move(), w.mAl); + obj.AddMember("sampler", tex.sampler->index, w.mAl); } } @@ -490,7 +472,7 @@ namespace glTF2 { // Add the target scene field if (mAsset.scene) { - mDoc.AddMember("scene", StringRef(mAsset.scene->id), mAl); + mDoc.AddMember("scene", mAsset.scene->index, mAl); } } @@ -582,7 +564,7 @@ namespace glTF2 { GLB_Header header; memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); - header.version = 1; + header.version = 2; AI_SWAP4(header.version); header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); @@ -624,8 +606,8 @@ namespace glTF2 { if (false) exts.PushBack(StringRef("KHR_binary_glTF"), mAl); - if (false) - exts.PushBack(StringRef("KHR_materials_common"), mAl); + // This is used to export common materials with GLTF 2. + exts.PushBack(StringRef("KHR_materials_common"), mAl); } if (!exts.Empty()) @@ -653,9 +635,9 @@ namespace glTF2 { } Value* dict; - if (!(dict = FindObject(*container, d.mDictId))) { - container->AddMember(StringRef(d.mDictId), Value().SetObject().Move(), mDoc.GetAllocator()); - dict = FindObject(*container, d.mDictId); + if (!(dict = FindArray(*container, d.mDictId))) { + container->AddMember(StringRef(d.mDictId), Value().SetArray().Move(), mDoc.GetAllocator()); + dict = FindArray(*container, d.mDictId); } for (size_t i = 0; i < d.mObjs.size(); ++i) { @@ -670,7 +652,7 @@ namespace glTF2 { Write(obj, *d.mObjs[i], *this); - dict->AddMember(StringRef(d.mObjs[i]->id), obj, mAl); + dict->PushBack(obj, mAl); } } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index e6dc2c4b1..ac727905b 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -434,7 +434,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, RefmName =====> skinRef->jointNames // Find the node with id = mName. Ref nodeRef = mAsset.nodes.Get(aib->mName.C_Str()); - nodeRef->jointName = nodeRef->id; + nodeRef->jointName = nodeRef->name; unsigned int jointNamesIndex; bool addJointToJointNames = true; @@ -744,15 +744,27 @@ void glTF2Exporter::ExportMeshes() skinRef->bindShapeMatrix.isPresent = true; IdentityMatrix4(skinRef->bindShapeMatrix.value); - // Find node that contains this mesh and add "skeletons" and "skin" attributes to that node. + // Find nodes that contain a mesh with bones and add "skeletons" and "skin" attributes to those nodes. Ref rootNode = mAsset->nodes.Get(unsigned(0)); Ref meshNode; - std::string meshID = mAsset->meshes.Get(unsigned(0))->id; - FindMeshNode(rootNode, meshNode, meshID); - - Ref rootJoint = FindSkeletonRootJoint(skinRef); - meshNode->skeletons.push_back(rootJoint); - meshNode->skin = skinRef; + for (unsigned int meshIndex = 0; meshIndex < mAsset->meshes.Size(); ++meshIndex) { + Ref mesh = mAsset->meshes.Get(meshIndex); + bool hasBones = false; + for (unsigned int i = 0; i < mesh->primitives.size(); ++i) { + if (!mesh->primitives[i].attributes.weight.empty()) { + hasBones = true; + break; + } + } + if (!hasBones) { + continue; + } + std::string meshID = mesh->id; + FindMeshNode(rootNode, meshNode, meshID); + Ref rootJoint = FindSkeletonRootJoint(skinRef); + meshNode->skeletons.push_back(rootJoint); + meshNode->skin = skinRef; + } } } @@ -787,9 +799,11 @@ unsigned int glTF2Exporter::ExportNodeHierarchy(const aiNode* n) */ unsigned int glTF2Exporter::ExportNode(const aiNode* n, Ref& parent) { - Ref node = mAsset->nodes.Create(mAsset->FindUniqueID(n->mName.C_Str(), "node")); + std::string name = mAsset->FindUniqueID(n->mName.C_Str(), "node"); + Ref node = mAsset->nodes.Create(name); node->parent = parent; + node->name = name; if (!n->mTransformation.IsIdentity()) { node->matrix.isPresent = true; @@ -826,7 +840,7 @@ void glTF2Exporter::ExportScene() void glTF2Exporter::ExportMetadata() { AssetMetadata& asset = mAsset->asset; - asset.version = 1; + asset.version = 2; char buffer[256]; ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", @@ -970,12 +984,12 @@ void glTF2Exporter::ExportAnimations() Animation::AnimChannel tmpAnimChannel; Animation::AnimSampler tmpAnimSampler; - tmpAnimChannel.sampler = name + "_" + channelType; + tmpAnimChannel.sampler = animRef->Samplers.size(); tmpAnimChannel.target.path = channelType; tmpAnimSampler.output = channelType; tmpAnimSampler.id = name + "_" + channelType; - tmpAnimChannel.target.id = mAsset->nodes.Get(nodeChannel->mNodeName.C_Str()); + tmpAnimChannel.target.node = mAsset->nodes.Get(nodeChannel->mNodeName.C_Str()); tmpAnimSampler.input = "TIME"; tmpAnimSampler.interpolation = "LINEAR"; From 16ed8861eb1ab7d03990006457ae8760b47cb713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20D=C3=A4hne?= Date: Wed, 26 Jul 2017 14:14:11 +0200 Subject: [PATCH 024/305] X3D importer: Workaround for buggy Android NDK (issue #1361) --- code/FIReader.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/FIReader.cpp b/code/FIReader.cpp index 19d94ed94..981d0b913 100755 --- a/code/FIReader.cpp +++ b/code/FIReader.cpp @@ -45,6 +45,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_X3D_IMPORTER +// Workaround for issue #1361 +// https://github.com/assimp/assimp/issues/1361 +#ifdef __ANDROID__ +#define _GLIBCXX_USE_C99 1 +#endif + #include "FIReader.hpp" #include "Exceptional.h" #include From acf8c54e559f936740c5ce38d278679b8ecb5dde Mon Sep 17 00:00:00 2001 From: jamesgk Date: Wed, 26 Jul 2017 14:20:53 -0700 Subject: [PATCH 025/305] glTF2: Fix animation export --- code/glTF2AssetWriter.inl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 2c3f2fffd..03e73eb27 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -113,13 +113,12 @@ namespace glTF2 { Value valChannel; valChannel.SetObject(); { - Animation::AnimSampler& s = a.Samplers[c.sampler]; - valChannel.AddMember("sampler", s.id, w.mAl); + valChannel.AddMember("sampler", c.sampler, w.mAl); Value valTarget; valTarget.SetObject(); { - valTarget.AddMember("id", StringRef(c.target.node->id), w.mAl); + valTarget.AddMember("node", c.target.node->index, w.mAl); valTarget.AddMember("path", c.target.path, w.mAl); } valChannel.AddMember("target", valTarget, w.mAl); From b7b17b03ec57ed81a9203ec9c3b40766b3eec22e Mon Sep 17 00:00:00 2001 From: jamesgk Date: Wed, 26 Jul 2017 14:39:35 -0700 Subject: [PATCH 026/305] glTF2: use opacity for diffuse alpha + alphaMode The "transparency" attribute is unused in glTF2. --- code/glTF2AssetWriter.inl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 03e73eb27..646b31ec5 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -212,17 +212,20 @@ namespace glTF2 { inline void Write(Value& obj, Material& m, AssetWriter& w) { + if (m.transparent) { + obj.AddMember("alphaMode", "BLEND", w.mAl); + } + Value v; v.SetObject(); { + if (m.transparent && !m.diffuse.texture) { + m.diffuse.color[3] = m.transparency; + } WriteColorOrTex(v, m.ambient, m.ambient.texture ? "ambientTexture" : "ambientFactor", w.mAl); WriteColorOrTex(v, m.diffuse, m.diffuse.texture ? "diffuseTexture" : "diffuseFactor", w.mAl); WriteColorOrTex(v, m.specular, m.specular.texture ? "specularTexture" : "specularFactor", w.mAl); WriteColorOrTex(v, m.emission, m.emission.texture ? "emissionTexture" : "emissionFactor", w.mAl); - - if (m.transparent) - v.AddMember("transparency", m.transparency, w.mAl); - v.AddMember("shininessFactor", m.shininess, w.mAl); } v.AddMember("type", "commonPhong", w.mAl); From c91e9a94daf563b8761d953bb4b7c4338b04e6e0 Mon Sep 17 00:00:00 2001 From: jamesgk Date: Wed, 26 Jul 2017 14:42:22 -0700 Subject: [PATCH 027/305] glTF2: export materials' normal maps --- code/glTF2Asset.h | 1 + code/glTF2AssetWriter.inl | 16 +++++++++--- code/glTF2Exporter.cpp | 51 +++++++++++++++++++++------------------ code/glTF2Exporter.h | 4 ++- 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 36caba2cb..bbef4413e 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -697,6 +697,7 @@ namespace glTF2 TexProperty diffuse; TexProperty specular; TexProperty emission; + Ref normal; bool doubleSided; bool transparent; diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 646b31ec5..6e1865bbf 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -196,14 +196,20 @@ namespace glTF2 { } namespace { - inline void WriteColorOrTex(Value& obj, TexProperty& prop, const char* propName, MemoryPoolAllocator<>& al) + inline void WriteTex(Value& obj, Ref texture, const char* propName, MemoryPoolAllocator<>& al) { - if (prop.texture) { + if (texture) { Value tex; tex.SetObject(); - tex.AddMember("index", prop.texture->index, al); + tex.AddMember("index", texture->index, al); obj.AddMember(StringRef(propName), tex, al); - } else { + } + } + + inline void WriteColorOrTex(Value& obj, TexProperty& prop, const char* propName, MemoryPoolAllocator<>& al) + { + WriteTex(obj, prop.texture, propName, al); + if (!prop.texture) { Value col; obj.AddMember(StringRef(propName), MakeValue(col, prop.color, al), al); } @@ -233,6 +239,8 @@ namespace glTF2 { ext.SetObject(); ext.AddMember("KHR_materials_common", v, w.mAl); obj.AddMember("extensions", ext, w.mAl); + + WriteTex(obj, m.normal, "normalTexture", w.mAl); } namespace { diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index ac727905b..97b60a604 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -240,10 +240,10 @@ namespace { } } -void glTF2Exporter::GetTexSampler(const aiMaterial* mat, TexProperty& prop) +void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture) { std::string samplerId = mAsset->FindUniqueID("", "sampler"); - prop.texture->sampler = mAsset->samplers.Create(samplerId); + texture->sampler = mAsset->samplers.Create(samplerId); aiTextureMapMode mapU, mapV; aiGetMaterialInteger(mat,AI_MATKEY_MAPPINGMODE_U_DIFFUSE(0),(int*)&mapU); @@ -251,45 +251,44 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, TexProperty& prop) switch (mapU) { case aiTextureMapMode_Wrap: - prop.texture->sampler->wrapS = SamplerWrap_Repeat; + texture->sampler->wrapS = SamplerWrap_Repeat; break; case aiTextureMapMode_Clamp: - prop.texture->sampler->wrapS = SamplerWrap_Clamp_To_Edge; + texture->sampler->wrapS = SamplerWrap_Clamp_To_Edge; break; case aiTextureMapMode_Mirror: - prop.texture->sampler->wrapS = SamplerWrap_Mirrored_Repeat; + texture->sampler->wrapS = SamplerWrap_Mirrored_Repeat; break; case aiTextureMapMode_Decal: default: - prop.texture->sampler->wrapS = SamplerWrap_Repeat; + texture->sampler->wrapS = SamplerWrap_Repeat; break; }; switch (mapV) { case aiTextureMapMode_Wrap: - prop.texture->sampler->wrapT = SamplerWrap_Repeat; + texture->sampler->wrapT = SamplerWrap_Repeat; break; case aiTextureMapMode_Clamp: - prop.texture->sampler->wrapT = SamplerWrap_Clamp_To_Edge; + texture->sampler->wrapT = SamplerWrap_Clamp_To_Edge; break; case aiTextureMapMode_Mirror: - prop.texture->sampler->wrapT = SamplerWrap_Mirrored_Repeat; + texture->sampler->wrapT = SamplerWrap_Mirrored_Repeat; break; case aiTextureMapMode_Decal: default: - prop.texture->sampler->wrapT = SamplerWrap_Repeat; + texture->sampler->wrapT = SamplerWrap_Repeat; break; }; // Hard coded Texture filtering options because I do not know where to find them in the aiMaterial. - prop.texture->sampler->magFilter = SamplerMagFilter_Linear; - prop.texture->sampler->minFilter = SamplerMinFilter_Linear; + texture->sampler->magFilter = SamplerMagFilter_Linear; + texture->sampler->minFilter = SamplerMinFilter_Linear; } -void glTF2Exporter::GetMatColorOrTex(const aiMaterial* mat, TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt) +void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt) { aiString tex; - aiColor4D col; if (mat->GetTextureCount(tt) > 0) { if (mat->Get(AI_MATKEY_TEXTURE(tt, 0), tex) == AI_SUCCESS) { std::string path = tex.C_Str(); @@ -298,43 +297,48 @@ void glTF2Exporter::GetMatColorOrTex(const aiMaterial* mat, TexProperty& prop, c if (path[0] != '*') { std::map::iterator it = mTexturesByPath.find(path); if (it != mTexturesByPath.end()) { - prop.texture = mAsset->textures.Get(it->second); + texture = mAsset->textures.Get(it->second); } } - if (!prop.texture) { + if (!texture) { std::string texId = mAsset->FindUniqueID("", "texture"); - prop.texture = mAsset->textures.Create(texId); - mTexturesByPath[path] = prop.texture.GetIndex(); + texture = mAsset->textures.Create(texId); + mTexturesByPath[path] = texture.GetIndex(); std::string imgId = mAsset->FindUniqueID("", "image"); - prop.texture->source = mAsset->images.Create(imgId); + texture->source = mAsset->images.Create(imgId); if (path[0] == '*') { // embedded aiTexture* tex = mScene->mTextures[atoi(&path[1])]; uint8_t* data = reinterpret_cast(tex->pcData); - prop.texture->source->SetData(data, tex->mWidth, *mAsset); + texture->source->SetData(data, tex->mWidth, *mAsset); if (tex->achFormatHint[0]) { std::string mimeType = "image/"; mimeType += (memcmp(tex->achFormatHint, "jpg", 3) == 0) ? "jpeg" : tex->achFormatHint; - prop.texture->source->mimeType = mimeType; + texture->source->mimeType = mimeType; } } else { - prop.texture->source->uri = path; + texture->source->uri = path; } - GetTexSampler(mat, prop); + GetTexSampler(mat, texture); } } } } +} +void glTF2Exporter::GetMatColorOrTex(const aiMaterial* mat, TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt) +{ + aiColor4D col; if (mat->Get(propName, type, idx, col) == AI_SUCCESS) { prop.color[0] = col.r; prop.color[1] = col.g; prop.color[2] = col.b; prop.color[3] = col.a; } + GetMatTex(mat, prop.texture, tt); } @@ -357,6 +361,7 @@ void glTF2Exporter::ExportMaterials() GetMatColorOrTex(mat, m->diffuse, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_DIFFUSE); GetMatColorOrTex(mat, m->specular, AI_MATKEY_COLOR_SPECULAR, aiTextureType_SPECULAR); GetMatColorOrTex(mat, m->emission, AI_MATKEY_COLOR_EMISSIVE, aiTextureType_EMISSIVE); + GetMatTex(mat, m->normal, aiTextureType_NORMALS); m->transparent = mat->Get(AI_MATKEY_OPACITY, m->transparency) == aiReturn_SUCCESS && m->transparency != 1.0; diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index 1378fa9f8..17cc9cfdc 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -67,6 +67,7 @@ namespace glTF2 class Asset; struct TexProperty; struct Node; + struct Texture; } namespace Assimp @@ -100,7 +101,8 @@ namespace Assimp void WriteBinaryData(IOStream* outfile, std::size_t sceneLength); - void GetTexSampler(const aiMaterial* mat, glTF2::TexProperty& prop); + void GetTexSampler(const aiMaterial* mat, glTF2::Ref texture); + void GetMatTex(const aiMaterial* mat, glTF2::Ref& texture, aiTextureType tt); void GetMatColorOrTex(const aiMaterial* mat, glTF2::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt); void ExportMetadata(); void ExportMaterials(); From d74306ad67d1ae13420a700ded2260c3615a47ed Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 27 Jul 2017 13:07:26 +0200 Subject: [PATCH 028/305] fix build --- code/FBXAnimation.cpp | 3 +-- test/unit/utProfiler.cpp | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/code/FBXAnimation.cpp b/code/FBXAnimation.cpp index 6b2c1eff6..e905b5bfc 100644 --- a/code/FBXAnimation.cpp +++ b/code/FBXAnimation.cpp @@ -50,12 +50,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "FBXDocument.h" #include "FBXImporter.h" #include "FBXDocumentUtil.h" -#include "FBXProperties.h" namespace Assimp { namespace FBX { - using namespace Util; +using namespace Util; // ------------------------------------------------------------------------------------------------ AnimationCurve::AnimationCurve(uint64_t id, const Element& element, const std::string& name, const Document& /*doc*/) diff --git a/test/unit/utProfiler.cpp b/test/unit/utProfiler.cpp index 0edd08255..75e25e5ba 100644 --- a/test/unit/utProfiler.cpp +++ b/test/unit/utProfiler.cpp @@ -60,7 +60,6 @@ public: virtual void TearDown() { DefaultLogger::get()->detatchStream( m_stream ); m_stream = nullptr; - DefaultLogger::kill(); } }; From 4b860dfc42e68c0098c0b5b348b3189c10c977c6 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 27 Jul 2017 13:31:22 +0200 Subject: [PATCH 029/305] fix build second try --- test/unit/utProfiler.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/utProfiler.cpp b/test/unit/utProfiler.cpp index 75e25e5ba..ea71dd688 100644 --- a/test/unit/utProfiler.cpp +++ b/test/unit/utProfiler.cpp @@ -51,7 +51,7 @@ class utProfiler : public ::testing::Test { public: LogStream *m_stream; - virtual void SetUp() { + /*virtual void SetUp() { m_stream = new UTLogStream; DefaultLogger::create(); DefaultLogger::get()->attachStream( m_stream ); @@ -60,7 +60,7 @@ public: virtual void TearDown() { DefaultLogger::get()->detatchStream( m_stream ); m_stream = nullptr; - } + }*/ }; TEST_F( utProfiler, addRegion_success ) { @@ -71,6 +71,6 @@ TEST_F( utProfiler, addRegion_success ) { j++; } myProfiler.EndRegion( "t1" ); - UTLogStream *stream( (UTLogStream*) m_stream ); + //UTLogStream *stream( (UTLogStream*) m_stream ); //EXPECT_FALSE( stream->m_messages.empty() ); } From a2bbf76cf4628d1ffd21b34654cd088b8c43ce1b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 27 Jul 2017 20:49:49 +0200 Subject: [PATCH 030/305] StreamReader: fix out-of-range exception --- code/IOStreamBuffer.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/code/IOStreamBuffer.h b/code/IOStreamBuffer.h index 6ca37b6eb..d7528de7e 100644 --- a/code/IOStreamBuffer.h +++ b/code/IOStreamBuffer.h @@ -284,11 +284,17 @@ bool IOStreamBuffer::getNextDataLine( std::vector &buffer, T continuationT return true; } +static +inline +bool isEndOfCache( size_t pos, size_t cacheSize ) { + return ( pos == cacheSize ); +} + template inline bool IOStreamBuffer::getNextLine(std::vector &buffer) { buffer.resize(m_cacheSize); - if (m_cachePos == m_cacheSize || 0 == m_filePos) { + if ( isEndOfCache( m_cachePos, m_cacheSize ) || 0 == m_filePos) { if (!readNextBlock()) { return false; } @@ -300,11 +306,16 @@ bool IOStreamBuffer::getNextLine(std::vector &buffer) { ++m_cachePos; } ++m_cachePos; + if ( isEndOfCache( m_cachePos, m_cacheSize ) ) { + if ( !readNextBlock() ) { + return false; + } + } } size_t i = 0; - while (!IsLineEnd(m_cache[m_cachePos])) { - buffer[i] = m_cache[m_cachePos]; + while (!IsLineEnd(m_cache[ m_cachePos ])) { + buffer[i] = m_cache[ m_cachePos ]; m_cachePos++; i++; if (m_cachePos >= m_cacheSize) { From 5939d811380fcdeea4b59ffa881517025bb382aa Mon Sep 17 00:00:00 2001 From: jamesgk Date: Thu, 27 Jul 2017 15:18:35 -0700 Subject: [PATCH 031/305] glTF2: Use better mipmap filter defaults --- code/glTF2Exporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 97b60a604..b6a2efdd0 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -283,7 +283,7 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture) // Hard coded Texture filtering options because I do not know where to find them in the aiMaterial. texture->sampler->magFilter = SamplerMagFilter_Linear; - texture->sampler->minFilter = SamplerMinFilter_Linear; + texture->sampler->minFilter = SamplerMinFilter_Linear_Mipmap_Linear; } void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt) From 6c526d7c645416cf294f2572d79b31242624956c Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 28 Jul 2017 22:23:08 +0200 Subject: [PATCH 032/305] PPdPmdParser: fix compilation for clang. --- code/MMDPmdParser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/MMDPmdParser.h b/code/MMDPmdParser.h index 44b64585a..5439b9043 100644 --- a/code/MMDPmdParser.h +++ b/code/MMDPmdParser.h @@ -158,7 +158,7 @@ namespace pmd sphere_filename.clear(); } else { - *pstar = (char)NULL; + *pstar = 0; texture_filename = std::string(buffer); sphere_filename = std::string(pstar+1); } From 3fe8c72f5d595d16dcc71852cfc759590e4873b9 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 28 Jul 2017 22:27:36 +0200 Subject: [PATCH 033/305] new release: update to 4.0.1. --- CMakeLists.txt | 2 +- code/Version.cpp | 7 +++---- include/assimp/version.h | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be7d46438..d781d1fd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,7 +106,7 @@ ENDIF(NOT BUILD_SHARED_LIBS) # Define here the needed parameters SET (ASSIMP_VERSION_MAJOR 4) SET (ASSIMP_VERSION_MINOR 0) -SET (ASSIMP_VERSION_PATCH 0) # subversion revision? +SET (ASSIMP_VERSION_PATCH 1) # subversion revision? SET (ASSIMP_VERSION ${ASSIMP_VERSION_MAJOR}.${ASSIMP_VERSION_MINOR}.${ASSIMP_VERSION_PATCH}) SET (ASSIMP_SOVERSION 4) SET (PROJECT_VERSION "${ASSIMP_VERSION}") diff --git a/code/Version.cpp b/code/Version.cpp index 4149f381a..2cd759817 100644 --- a/code/Version.cpp +++ b/code/Version.cpp @@ -107,8 +107,7 @@ ASSIMP_API unsigned int aiGetCompileFlags () { #include "revision.h" // ------------------------------------------------------------------------------------------------ -ASSIMP_API unsigned int aiGetVersionRevision () -{ +ASSIMP_API unsigned int aiGetVersionRevision() { return GitVersion; } @@ -133,8 +132,7 @@ ASSIMP_API aiScene::aiScene() } // ------------------------------------------------------------------------------------------------ -ASSIMP_API aiScene::~aiScene() -{ +ASSIMP_API aiScene::~aiScene() { // delete all sub-objects recursively delete mRootNode; @@ -173,3 +171,4 @@ ASSIMP_API aiScene::~aiScene() delete static_cast( mPrivate ); } + diff --git a/include/assimp/version.h b/include/assimp/version.h index b85e6ccc8..d1821fa2b 100644 --- a/include/assimp/version.h +++ b/include/assimp/version.h @@ -105,3 +105,4 @@ ASSIMP_API unsigned int aiGetCompileFlags (void); #endif #endif // !! #ifndef AI_VERSION_H_INC + From eb8639d7c8a056d1c1b1cd42598c84ba3974c41b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 28 Jul 2017 22:33:41 +0200 Subject: [PATCH 034/305] CHANGES: add documentation of featureset for 4.0.1 --- CHANGES | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGES b/CHANGES index d21042b89..d5faab2e0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,18 @@ ---------------------------------------------------------------------- CHANGELOG ---------------------------------------------------------------------- +4.0.1 (2017-07-28) + - FIXES/HOUSEKEEPING: + - fix version test. + - Not compiling when using ASSIMP_DOUBLE_PRECISION + - Added support for python3 + - Check if cmake is installed with brew + - Low performance in OptimizeMeshesProcess::ProcessNode with huge numbers of meshes + - Elapsed seconds not shown correctly + - StreamReader: fix out-of-range exception + - PPdPmdParser: fix compilation for clang + + 4.0.0 (2017-07-18) FEATURES: From 7557fdbb72138ad0a4158e7290d2e11597622c95 Mon Sep 17 00:00:00 2001 From: Date: Sat, 29 Jul 2017 07:45:25 -0400 Subject: [PATCH 035/305] Fix install for builds with MSVC compiler and NMake. The CMake variable "MSVC" and the related versioned variables indicate that the Visual Studio compiler is used but when using that compiler and the "NMake Makefiles JOM" build generator the files it tried to install weren't in the expected Debug folder. --- CMakeLists.txt | 27 +++++++++++++++++++-------- code/CMakeLists.txt | 27 +++++++++++++++++++-------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d781d1fd7..1c9821896 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -472,13 +472,24 @@ if(WIN32) if(MSVC12 OR MSVC14) add_custom_target(UpdateAssimpLibsDebugSymbolsAndDLLs COMMENT "Copying Assimp Libraries ..." VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) + IF(CMAKE_GENERATOR MATCHES "^Visual Studio") + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) + ELSE() + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) + add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) + ENDIF() ENDIF(MSVC12 OR MSVC14) ENDIF (WIN32) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 0477f5fef..43e8ccf36 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -934,14 +934,25 @@ if (ASSIMP_ANDROID_JNIIOSYSTEM) endif(ASSIMP_ANDROID_JNIIOSYSTEM) if(MSVC AND ASSIMP_INSTALL_PDB) - install(FILES ${Assimp_BINARY_DIR}/code/Debug/assimp${LIBRARY_SUFFIX}${CMAKE_DEBUG_POSTFIX}.pdb - DESTINATION ${ASSIMP_LIB_INSTALL_DIR} - CONFIGURATIONS Debug - ) - install(FILES ${Assimp_BINARY_DIR}/code/RelWithDebInfo/assimp${LIBRARY_SUFFIX}.pdb - DESTINATION ${ASSIMP_LIB_INSTALL_DIR} - CONFIGURATIONS RelWithDebInfo - ) + IF(CMAKE_GENERATOR MATCHES "^Visual Studio") + install(FILES ${Assimp_BINARY_DIR}/code/Debug/assimp${LIBRARY_SUFFIX}${CMAKE_DEBUG_POSTFIX}.pdb + DESTINATION ${ASSIMP_LIB_INSTALL_DIR} + CONFIGURATIONS Debug + ) + install(FILES ${Assimp_BINARY_DIR}/code/RelWithDebInfo/assimp${LIBRARY_SUFFIX}.pdb + DESTINATION ${ASSIMP_LIB_INSTALL_DIR} + CONFIGURATIONS RelWithDebInfo + ) + ELSE() + install(FILES ${Assimp_BINARY_DIR}/code/assimp${LIBRARY_SUFFIX}${CMAKE_DEBUG_POSTFIX}.pdb + DESTINATION ${ASSIMP_LIB_INSTALL_DIR} + CONFIGURATIONS Debug + ) + install(FILES ${Assimp_BINARY_DIR}/code/assimp${LIBRARY_SUFFIX}.pdb + DESTINATION ${ASSIMP_LIB_INSTALL_DIR} + CONFIGURATIONS RelWithDebInfo + ) + ENDIF() endif () if (ASSIMP_COVERALLS) From 5fe45a1aa34577d7af092942c2db3dadd24852a5 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 29 Jul 2017 16:45:50 +0200 Subject: [PATCH 036/305] Readme: update list of supported file formats. --- Readme.md | 78 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/Readme.md b/Readme.md index 1e8e66b92..4eb884c63 100644 --- a/Readme.md +++ b/Readme.md @@ -1,5 +1,6 @@ Open Asset Import Library (assimp) ================================== +A library to import and export various 3d-model-formats including scene-post-processing to generate missing render data. ### Current build status ### [![Linux Build Status](https://travis-ci.org/assimp/assimp.svg)](https://travis-ci.org/assimp/assimp) [![Windows Build Status](https://ci.appveyor.com/api/projects/status/tmo433wax6u6cjp4?svg=true)](https://ci.appveyor.com/project/kimkulling/assimp) @@ -29,46 +30,62 @@ Please check our Wiki as well: https://github.com/assimp/assimp/wiki A full list [is here](http://assimp.org/main_features_formats.html). __Importers__: - +- 3D - 3DS -- BLEND (Blender) -- DAE/Collada -- FBX -- IFC-STEP +- 3MF +- AC +- AC3D +- ACC +- AMJ - ASE +- ASK +- B3D; +- BLEND (Blender) +- BVH +- COB +- CMS +- DAE/Collada - DXF -- HMP +- ENFF +- FBX +- GLB/GLTF +- HMB +- IFC-STEP +- IRR / IRRMESH +- LWO +- LWS +- LXO - MD2 - MD3 - MD5 - MDC - MDL -- NFF -- PLY -- STL -- X -- OBJ -- OpenGEX -- SMD -- LWO -- LXO -- LWS -- TER -- AC3D +- MESH / MESH.XML +- MOT - MS3D -- COB -- Q3BSP -- XGL -- CSM -- BVH -- B3D - NDO -- Ogre Binary -- Ogre XML -- Q3D -- ASSBIN (Assimp custom format) -- glTF (partial) -- 3MF +- NFF +- OBJ +- OFF +- OGEX +- PLY +- PMX +- PRJ +- Q3O +- Q3S +- RAW +- SCN +- SIB +- SMD +- STL +- STP +- TER +- UC +- VTA +- X +- X3D +- XGL +- ZGL Additionally, some formats are supported by dependency on non-free code or external SDKs (not built by default): @@ -86,6 +103,7 @@ __Exporters__: - ASSBIN - STEP - glTF (partial) +- glTF2.0 ### Building ### Take a look into the `INSTALL` file. Our build system is CMake, if you used CMake before there is a good chance you know what to do. From 919f2a1ea969d46ac329afa736869d3a374e0ba1 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sun, 30 Jul 2017 21:21:51 +0200 Subject: [PATCH 037/305] Commandline tool: fix url to project space. --- tools/assimp_cmd/Main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/assimp_cmd/Main.cpp b/tools/assimp_cmd/Main.cpp index 5bf3b9f06..587e111d5 100644 --- a/tools/assimp_cmd/Main.cpp +++ b/tools/assimp_cmd/Main.cpp @@ -48,7 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. const char* AICMD_MSG_ABOUT = "------------------------------------------------------ \n" -"Open Asset Import Library (\"Assimp\", http://assimp.sourceforge.net) \n" +"Open Asset Import Library (\"Assimp\", https://github.com/assimp/assimp) \n" " -- Commandline toolchain --\n" "------------------------------------------------------ \n\n" From 232954c15e390bff78123408e4df64e190af02f2 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Mon, 31 Jul 2017 14:25:00 +0200 Subject: [PATCH 038/305] cmake cleanup. --- CMakeLists.txt | 104 ++++++++++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c9821896..676a48385 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #---------------------------------------------------------------------- SET(CMAKE_LEGACY_CYGWIN_WIN32 0) # Remove when CMake >= 2.8.4 is required -cmake_minimum_required( VERSION 2.8 ) +CMAKE_MINIMUM_REQUIRED( VERSION 2.8 ) PROJECT( Assimp ) # All supported options ############################################### @@ -62,11 +62,11 @@ OPTION( ASSIMP_BUILD_ZLIB "Build your own zlib" OFF ) -option( ASSIMP_BUILD_ASSIMP_TOOLS +OPTION( ASSIMP_BUILD_ASSIMP_TOOLS "If the supplementary tools for Assimp are built in addition to the library." ON ) -option ( ASSIMP_BUILD_SAMPLES +OPTION ( ASSIMP_BUILD_SAMPLES "If the official samples are built as well (needs Glut)." OFF ) @@ -75,24 +75,24 @@ OPTION ( ASSIMP_BUILD_TESTS ON ) OPTION ( ASSIMP_COVERALLS - "Eańable this to measure test coverage." - OFF + "Enable this to measure test coverage." + OFF ) -option ( SYSTEM_IRRXML - "Use system installed Irrlicht/IrrXML library." - OFF +OPTION ( SYSTEM_IRRXML + "Use system installed Irrlicht/IrrXML library." + OFF ) OPTION ( BUILD_DOCS - "Build documentation using Doxygen." - OFF + "Build documentation using Doxygen." + OFF ) if (WIN32) - add_definitions( -DWIN32_LEAN_AND_MEAN ) + ADD_DEFINITIONS( -DWIN32_LEAN_AND_MEAN ) endif() IF(MSVC) - set (CMAKE_PREFIX_PATH "D:\\libs\\devil") + SET (CMAKE_PREFIX_PATH "D:\\libs\\devil") OPTION( ASSIMP_INSTALL_PDB "Install MSVC debug files." ON @@ -106,15 +106,15 @@ ENDIF(NOT BUILD_SHARED_LIBS) # Define here the needed parameters SET (ASSIMP_VERSION_MAJOR 4) SET (ASSIMP_VERSION_MINOR 0) -SET (ASSIMP_VERSION_PATCH 1) # subversion revision? +SET (ASSIMP_VERSION_PATCH 1) SET (ASSIMP_VERSION ${ASSIMP_VERSION_MAJOR}.${ASSIMP_VERSION_MINOR}.${ASSIMP_VERSION_PATCH}) SET (ASSIMP_SOVERSION 4) SET (PROJECT_VERSION "${ASSIMP_VERSION}") -SET(ASSIMP_PACKAGE_VERSION "0" CACHE STRING "the package-specific version used for uploading the sources") +SET( ASSIMP_PACKAGE_VERSION "0" CACHE STRING "the package-specific version used for uploading the sources" ) # Needed for openddl_parser config, no use of c++11 at this moment -add_definitions( -DOPENDDL_NO_USE_CPP11 ) +ADD_DEFINITIONS( -DOPENDDL_NO_USE_CPP11 ) set_property( GLOBAL PROPERTY CXX_STANDARD 11 ) # Get the current working branch @@ -136,32 +136,33 @@ EXECUTE_PROCESS( ) IF(NOT GIT_COMMIT_HASH) - SET(GIT_COMMIT_HASH 0) + SET(GIT_COMMIT_HASH 0) ENDIF(NOT GIT_COMMIT_HASH) IF(ASSIMP_DOUBLE_PRECISION) - ADD_DEFINITIONS(-DASSIMP_DOUBLE_PRECISION) + ADD_DEFINITIONS(-DASSIMP_DOUBLE_PRECISION) ENDIF(ASSIMP_DOUBLE_PRECISION) # Check for OpenMP support find_package(OpenMP) if (OPENMP_FOUND) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() -configure_file( +CONFIGURE_FILE( ${CMAKE_CURRENT_LIST_DIR}/revision.h.in ${CMAKE_CURRENT_BINARY_DIR}/revision.h ) -configure_file( +CONFIGURE_FILE( ${CMAKE_CURRENT_LIST_DIR}/include/assimp/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/assimp/config.h ) -include_directories( +INCLUDE_DIRECTORIES( ./ + include ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/include ) @@ -179,7 +180,7 @@ IF( UNIX ) ENDIF() # Use GNUInstallDirs for Unix predefined directories - include(GNUInstallDirs) + INCLUDE(GNUInstallDirs) ENDIF( UNIX ) @@ -193,20 +194,18 @@ ELSEIF(MSVC) # enable multi-core compilation with MSVC add_compile_options(/MP) ELSEIF ( "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" ) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fvisibility=hidden -fPIC -Wall -Wno-long-long -pedantic -std=c++11" ) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fvisibility=hidden -fPIC -Wall -Wno-long-long -pedantic -std=c++11" ) ELSEIF( CMAKE_COMPILER_IS_MINGW ) SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -Wall -Wno-long-long -pedantic -std=c++11" ) - add_definitions( -U__STRICT_ANSI__ ) + ADD_DEFINITIONS( -U__STRICT_ANSI__ ) ENDIF() if (ASSIMP_COVERALLS) - include(Coveralls) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + INCLUDE(Coveralls) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") endif() -INCLUDE_DIRECTORIES( include ) - INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) @@ -237,13 +236,13 @@ ENDIF() # Only generate this target if no higher-level project already has IF (NOT TARGET uninstall) # add make uninstall capability - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) + CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) add_custom_target(uninstall "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake") ENDIF() # cmake configuration files -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/assimp-config.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/assimp-config.cmake" @ONLY IMMEDIATE) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/assimp-config-version.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/assimp-config-version.cmake" @ONLY IMMEDIATE) +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/assimp-config.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/assimp-config.cmake" @ONLY IMMEDIATE) +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/assimp-config-version.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/assimp-config-version.cmake" @ONLY IMMEDIATE) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/assimp-config.cmake" "${CMAKE_CURRENT_BINARY_DIR}/assimp-config-version.cmake" DESTINATION "${ASSIMP_LIB_INSTALL_DIR}/cmake/assimp-${ASSIMP_VERSION_MAJOR}.${ASSIMP_VERSION_MINOR}" COMPONENT ${LIBASSIMP-DEV_COMPONENT}) FIND_PACKAGE( DirectX ) @@ -265,9 +264,9 @@ ENDIF( NOT ASSIMP_BUILD_ZLIB ) IF( NOT ZLIB_FOUND ) message(STATUS "compiling zlib from souces") - include(CheckIncludeFile) - include(CheckTypeSize) - include(CheckFunctionExists) + INCLUDE(CheckIncludeFile) + INCLUDE(CheckTypeSize) + INCLUDE(CheckFunctionExists) # compile from sources add_subdirectory(contrib/zlib) SET(ZLIB_FOUND 1) @@ -473,23 +472,24 @@ if(WIN32) if(MSVC12 OR MSVC14) add_custom_target(UpdateAssimpLibsDebugSymbolsAndDLLs COMMENT "Copying Assimp Libraries ..." VERBATIM) IF(CMAKE_GENERATOR MATCHES "^Visual Studio") - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Release/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/Debug/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) ELSE() - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) - add_custom_command(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.dll VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.exp VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mt.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mt.lib VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.dll ${BIN_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.dll VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.exp ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.exp VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.ilk VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.lib ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.lib VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) + ADD_CUSTOM_COMMAND(TARGET UpdateAssimpLibsDebugSymbolsAndDLLs COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/code/assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb ${LIB_DIR}assimp-${ASSIMP_MSVC_VERSION}-mtd.pdb VERBATIM) ENDIF() ENDIF(MSVC12 OR MSVC14) ENDIF (WIN32) From 81fd027860441ea539e054128b59623b89273bbc Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Wed, 2 Aug 2017 11:35:57 +0200 Subject: [PATCH 039/305] closes https://github.com/assimp/assimp/issues/1270: use HasTexture for texture coordinates. --- code/PlyLoader.cpp | 2 +- code/ValidateDataStructure.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/PlyLoader.cpp b/code/PlyLoader.cpp index 0e99df7a9..937e839f5 100644 --- a/code/PlyLoader.cpp +++ b/code/PlyLoader.cpp @@ -537,7 +537,7 @@ void PLYImporter::LoadVertex(const PLY::Element* pcElement, const PLY::ElementIn if (haveTextureCoords) { - if (mGeneratedMesh->mTextureCoords[0] == NULL) + if (mGeneratedMesh->HasTextureCoords(0)) { mGeneratedMesh->mNumUVComponents[0] = 2; mGeneratedMesh->mTextureCoords[0] = new aiVector3D[mGeneratedMesh->mNumVertices]; diff --git a/code/ValidateDataStructure.cpp b/code/ValidateDataStructure.cpp index a536058bc..ae1e0d342 100644 --- a/code/ValidateDataStructure.cpp +++ b/code/ValidateDataStructure.cpp @@ -409,12 +409,12 @@ void ValidateDSProcess::Validate( const aiMesh* pMesh) // the MSB flag is temporarily used by the extra verbose // mode to tell us that the JoinVerticesProcess might have // been executed already. - if ( !(this->mScene->mFlags & AI_SCENE_FLAGS_NON_VERBOSE_FORMAT ) && !(this->mScene->mFlags & AI_SCENE_FLAGS_ALLOW_SHARED) && + /*if ( !(this->mScene->mFlags & AI_SCENE_FLAGS_NON_VERBOSE_FORMAT ) && !(this->mScene->mFlags & AI_SCENE_FLAGS_ALLOW_SHARED) && abRefList[face.mIndices[a]]) { ReportError("aiMesh::mVertices[%i] is referenced twice - second " "time by aiMesh::mFaces[%i]::mIndices[%i]",face.mIndices[a],i,a); - } + }*/ abRefList[face.mIndices[a]] = true; } } From def42bf624c99df8747f3ab9d065374e995b2617 Mon Sep 17 00:00:00 2001 From: Knox Shao <0xcccc@outlook.com> Date: Wed, 2 Aug 2017 19:15:07 +0800 Subject: [PATCH 040/305] Fix android build issues --- code/D3MFOpcPackage.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/code/D3MFOpcPackage.cpp b/code/D3MFOpcPackage.cpp index 562b891be..ff6fddcd1 100644 --- a/code/D3MFOpcPackage.cpp +++ b/code/D3MFOpcPackage.cpp @@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include #include From 8478df7dbd19949cafb6b9c4d60be6f0abe1475a Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Wed, 2 Aug 2017 14:24:53 +0200 Subject: [PATCH 041/305] PlyLoader: fix vertex attribute lookup. --- code/PlyLoader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/PlyLoader.cpp b/code/PlyLoader.cpp index 937e839f5..b9af10c0e 100644 --- a/code/PlyLoader.cpp +++ b/code/PlyLoader.cpp @@ -513,7 +513,7 @@ void PLYImporter::LoadVertex(const PLY::Element* pcElement, const PLY::ElementIn mGeneratedMesh->mMaterialIndex = 0; } - if (mGeneratedMesh->mVertices == NULL) + if (mGeneratedMesh->HasPositions() ) { mGeneratedMesh->mNumVertices = pcElement->NumOccur; mGeneratedMesh->mVertices = new aiVector3D[mGeneratedMesh->mNumVertices]; @@ -523,14 +523,14 @@ void PLYImporter::LoadVertex(const PLY::Element* pcElement, const PLY::ElementIn if (haveNormal) { - if (mGeneratedMesh->mNormals == NULL) + if (mGeneratedMesh->HasNormals() ) mGeneratedMesh->mNormals = new aiVector3D[mGeneratedMesh->mNumVertices]; mGeneratedMesh->mNormals[pos] = nOut; } if (haveColor) { - if (mGeneratedMesh->mColors[0] == NULL) + if (mGeneratedMesh->HasVertexColors( 0 ) ) mGeneratedMesh->mColors[0] = new aiColor4D[mGeneratedMesh->mNumVertices]; mGeneratedMesh->mColors[0][pos] = cOut; } From ba658e7813722445b69359065ee78d2aa9ef0bea Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 3 Aug 2017 14:57:48 +0200 Subject: [PATCH 042/305] ply-importer: fix creation of vertex attributes. --- code/PlyLoader.cpp | 391 ++++++++++++++------------------ test/unit/utPLYImportExport.cpp | 6 + 2 files changed, 178 insertions(+), 219 deletions(-) diff --git a/code/PlyLoader.cpp b/code/PlyLoader.cpp index b9af10c0e..090473893 100644 --- a/code/PlyLoader.cpp +++ b/code/PlyLoader.cpp @@ -299,252 +299,205 @@ void PLYImporter::InternReadFile(const std::string& pFile, } } -void PLYImporter::LoadVertex(const PLY::Element* pcElement, const PLY::ElementInstance* instElement, unsigned int pos) -{ - ai_assert(NULL != pcElement); - ai_assert(NULL != instElement); +void PLYImporter::LoadVertex(const PLY::Element* pcElement, const PLY::ElementInstance* instElement, unsigned int pos) { + ai_assert(NULL != pcElement); + ai_assert(NULL != instElement); - ai_uint aiPositions[3] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; - PLY::EDataType aiTypes[3] = { EDT_Char, EDT_Char, EDT_Char }; + ai_uint aiPositions[3] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; + PLY::EDataType aiTypes[3] = { EDT_Char, EDT_Char, EDT_Char }; - ai_uint aiNormal[3] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; - PLY::EDataType aiNormalTypes[3] = { EDT_Char, EDT_Char, EDT_Char }; + ai_uint aiNormal[3] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; + PLY::EDataType aiNormalTypes[3] = { EDT_Char, EDT_Char, EDT_Char }; - unsigned int aiColors[4] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; - PLY::EDataType aiColorsTypes[4] = { EDT_Char, EDT_Char, EDT_Char, EDT_Char }; + unsigned int aiColors[4] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; + PLY::EDataType aiColorsTypes[4] = { EDT_Char, EDT_Char, EDT_Char, EDT_Char }; - unsigned int aiTexcoord[2] = { 0xFFFFFFFF, 0xFFFFFFFF }; - PLY::EDataType aiTexcoordTypes[2] = { EDT_Char, EDT_Char }; + unsigned int aiTexcoord[2] = { 0xFFFFFFFF, 0xFFFFFFFF }; + PLY::EDataType aiTexcoordTypes[2] = { EDT_Char, EDT_Char }; - unsigned int cnt = 0; + // now check whether which normal components are available + unsigned int _a( 0 ), cnt( 0 ); + for ( std::vector::const_iterator a = pcElement->alProperties.begin(); + a != pcElement->alProperties.end(); ++a, ++_a) { + if ((*a).bIsList) { + continue; + } - // now check whether which normal components are available - unsigned int _a = 0; - for (std::vector::const_iterator a = pcElement->alProperties.begin(); - a != pcElement->alProperties.end(); ++a, ++_a) - { - if ((*a).bIsList)continue; - - // Positions - if (PLY::EST_XCoord == (*a).Semantic) - { - cnt++; - aiPositions[0] = _a; - aiTypes[0] = (*a).eType; - } - else if (PLY::EST_YCoord == (*a).Semantic) - { - cnt++; - aiPositions[1] = _a; - aiTypes[1] = (*a).eType; - } - else if (PLY::EST_ZCoord == (*a).Semantic) - { - cnt++; - aiPositions[2] = _a; - aiTypes[2] = (*a).eType; + // Positions + if (PLY::EST_XCoord == (*a).Semantic) { + ++cnt; + aiPositions[0] = _a; + aiTypes[0] = (*a).eType; + } else if (PLY::EST_YCoord == (*a).Semantic) { + ++cnt; + aiPositions[1] = _a; + aiTypes[1] = (*a).eType; + } else if (PLY::EST_ZCoord == (*a).Semantic) { + ++cnt; + aiPositions[2] = _a; + aiTypes[2] = (*a).eType; + } else if (PLY::EST_XNormal == (*a).Semantic) { + // Normals + ++cnt; + aiNormal[0] = _a; + aiNormalTypes[0] = (*a).eType; + } else if (PLY::EST_YNormal == (*a).Semantic) { + ++cnt; + aiNormal[1] = _a; + aiNormalTypes[1] = (*a).eType; + } else if (PLY::EST_ZNormal == (*a).Semantic) { + ++cnt; + aiNormal[2] = _a; + aiNormalTypes[2] = (*a).eType; + } else if (PLY::EST_Red == (*a).Semantic) { + // Colors + ++cnt; + aiColors[0] = _a; + aiColorsTypes[0] = (*a).eType; + } else if (PLY::EST_Green == (*a).Semantic) { + ++cnt; + aiColors[1] = _a; + aiColorsTypes[1] = (*a).eType; + } else if (PLY::EST_Blue == (*a).Semantic) { + ++cnt; + aiColors[2] = _a; + aiColorsTypes[2] = (*a).eType; + } else if (PLY::EST_Alpha == (*a).Semantic) { + ++cnt; + aiColors[3] = _a; + aiColorsTypes[3] = (*a).eType; + } else if (PLY::EST_UTextureCoord == (*a).Semantic) { + // Texture coordinates + ++cnt; + aiTexcoord[0] = _a; + aiTexcoordTypes[0] = (*a).eType; + } else if (PLY::EST_VTextureCoord == (*a).Semantic) { + ++cnt; + aiTexcoord[1] = _a; + aiTexcoordTypes[1] = (*a).eType; + } } - // Normals - else if (PLY::EST_XNormal == (*a).Semantic) - { - cnt++; - aiNormal[0] = _a; - aiNormalTypes[0] = (*a).eType; - } - else if (PLY::EST_YNormal == (*a).Semantic) - { - cnt++; - aiNormal[1] = _a; - aiNormalTypes[1] = (*a).eType; - } - else if (PLY::EST_ZNormal == (*a).Semantic) - { - cnt++; - aiNormal[2] = _a; - aiNormalTypes[2] = (*a).eType; - } - // Colors - else if (PLY::EST_Red == (*a).Semantic) - { - cnt++; - aiColors[0] = _a; - aiColorsTypes[0] = (*a).eType; - } - else if (PLY::EST_Green == (*a).Semantic) - { - cnt++; - aiColors[1] = _a; - aiColorsTypes[1] = (*a).eType; - } - else if (PLY::EST_Blue == (*a).Semantic) - { - cnt++; - aiColors[2] = _a; - aiColorsTypes[2] = (*a).eType; - } - else if (PLY::EST_Alpha == (*a).Semantic) - { - cnt++; - aiColors[3] = _a; - aiColorsTypes[3] = (*a).eType; - } - // Texture coordinates - else if (PLY::EST_UTextureCoord == (*a).Semantic) - { - cnt++; - aiTexcoord[0] = _a; - aiTexcoordTypes[0] = (*a).eType; - } - else if (PLY::EST_VTextureCoord == (*a).Semantic) - { - cnt++; - aiTexcoord[1] = _a; - aiTexcoordTypes[1] = (*a).eType; - } - } + // check whether we have a valid source for the vertex data + if (0 != cnt) { + // Position + aiVector3D vOut; + if (0xFFFFFFFF != aiPositions[0]) { + vOut.x = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiPositions[0]).avList.front(), aiTypes[0]); + } - // check whether we have a valid source for the vertex data - if (0 != cnt) - { - // Position - aiVector3D vOut; - if (0xFFFFFFFF != aiPositions[0]) - { - vOut.x = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiPositions[0]).avList.front(), aiTypes[0]); - } + if (0xFFFFFFFF != aiPositions[1]) { + vOut.y = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiPositions[1]).avList.front(), aiTypes[1]); + } - if (0xFFFFFFFF != aiPositions[1]) - { - vOut.y = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiPositions[1]).avList.front(), aiTypes[1]); - } + if (0xFFFFFFFF != aiPositions[2]) { + vOut.z = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiPositions[2]).avList.front(), aiTypes[2]); + } - if (0xFFFFFFFF != aiPositions[2]) - { - vOut.z = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiPositions[2]).avList.front(), aiTypes[2]); - } + // Normals + aiVector3D nOut; + bool haveNormal = false; + if (0xFFFFFFFF != aiNormal[0]) { + nOut.x = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiNormal[0]).avList.front(), aiNormalTypes[0]); + haveNormal = true; + } - // Normals - aiVector3D nOut; - bool haveNormal = false; - if (0xFFFFFFFF != aiNormal[0]) - { - nOut.x = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiNormal[0]).avList.front(), aiNormalTypes[0]); - haveNormal = true; - } + if (0xFFFFFFFF != aiNormal[1]) { + nOut.y = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiNormal[1]).avList.front(), aiNormalTypes[1]); + haveNormal = true; + } - if (0xFFFFFFFF != aiNormal[1]) - { - nOut.y = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiNormal[1]).avList.front(), aiNormalTypes[1]); - haveNormal = true; - } + if (0xFFFFFFFF != aiNormal[2]) { + nOut.z = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiNormal[2]).avList.front(), aiNormalTypes[2]); + haveNormal = true; + } - if (0xFFFFFFFF != aiNormal[2]) - { - nOut.z = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiNormal[2]).avList.front(), aiNormalTypes[2]); - haveNormal = true; - } + //Colors + aiColor4D cOut; + bool haveColor = false; + if (0xFFFFFFFF != aiColors[0]) { + cOut.r = NormalizeColorValue(GetProperty(instElement->alProperties, + aiColors[0]).avList.front(), aiColorsTypes[0]); + haveColor = true; + } - //Colors - aiColor4D cOut; - bool haveColor = false; - if (0xFFFFFFFF != aiColors[0]) - { - cOut.r = NormalizeColorValue(GetProperty(instElement->alProperties, - aiColors[0]).avList.front(), aiColorsTypes[0]); - haveColor = true; - } + if (0xFFFFFFFF != aiColors[1]) { + cOut.g = NormalizeColorValue(GetProperty(instElement->alProperties, + aiColors[1]).avList.front(), aiColorsTypes[1]); + haveColor = true; + } - if (0xFFFFFFFF != aiColors[1]) - { - cOut.g = NormalizeColorValue(GetProperty(instElement->alProperties, - aiColors[1]).avList.front(), aiColorsTypes[1]); - haveColor = true; - } + if (0xFFFFFFFF != aiColors[2]) { + cOut.b = NormalizeColorValue(GetProperty(instElement->alProperties, + aiColors[2]).avList.front(), aiColorsTypes[2]); + haveColor = true; + } - if (0xFFFFFFFF != aiColors[2]) - { - cOut.b = NormalizeColorValue(GetProperty(instElement->alProperties, - aiColors[2]).avList.front(), aiColorsTypes[2]); - haveColor = true; - } + // assume 1.0 for the alpha channel ifit is not set + if (0xFFFFFFFF == aiColors[3]) { + cOut.a = 1.0; + } else { + cOut.a = NormalizeColorValue(GetProperty(instElement->alProperties, + aiColors[3]).avList.front(), aiColorsTypes[3]); - // assume 1.0 for the alpha channel ifit is not set - if (0xFFFFFFFF == aiColors[3]) - { - cOut.a = 1.0; - } - else - { - cOut.a = NormalizeColorValue(GetProperty(instElement->alProperties, - aiColors[3]).avList.front(), aiColorsTypes[3]); + haveColor = true; + } - haveColor = true; - } + //Texture coordinates + aiVector3D tOut; + tOut.z = 0; + bool haveTextureCoords = false; + if (0xFFFFFFFF != aiTexcoord[0]) { + tOut.x = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiTexcoord[0]).avList.front(), aiTexcoordTypes[0]); + haveTextureCoords = true; + } - //Texture coordinates - aiVector3D tOut; - tOut.z = 0; - bool haveTextureCoords = false; - if (0xFFFFFFFF != aiTexcoord[0]) - { - tOut.x = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiTexcoord[0]).avList.front(), aiTexcoordTypes[0]); - haveTextureCoords = true; - } + if (0xFFFFFFFF != aiTexcoord[1]) { + tOut.y = PLY::PropertyInstance::ConvertTo( + GetProperty(instElement->alProperties, aiTexcoord[1]).avList.front(), aiTexcoordTypes[1]); + haveTextureCoords = true; + } - if (0xFFFFFFFF != aiTexcoord[1]) - { - tOut.y = PLY::PropertyInstance::ConvertTo( - GetProperty(instElement->alProperties, aiTexcoord[1]).avList.front(), aiTexcoordTypes[1]); - haveTextureCoords = true; - } + //create aiMesh if needed + if ( nullptr == mGeneratedMesh ) { + mGeneratedMesh = new aiMesh(); + mGeneratedMesh->mMaterialIndex = 0; + } - //create aiMesh if needed - if (mGeneratedMesh == NULL) - { - mGeneratedMesh = new aiMesh(); - mGeneratedMesh->mMaterialIndex = 0; - } + if (nullptr == mGeneratedMesh->mVertices) { + mGeneratedMesh->mNumVertices = pcElement->NumOccur; + mGeneratedMesh->mVertices = new aiVector3D[mGeneratedMesh->mNumVertices]; + } - if (mGeneratedMesh->HasPositions() ) - { - mGeneratedMesh->mNumVertices = pcElement->NumOccur; - mGeneratedMesh->mVertices = new aiVector3D[mGeneratedMesh->mNumVertices]; - } + mGeneratedMesh->mVertices[pos] = vOut; - mGeneratedMesh->mVertices[pos] = vOut; + if (haveNormal) { + if (nullptr == mGeneratedMesh->mNormals) + mGeneratedMesh->mNormals = new aiVector3D[mGeneratedMesh->mNumVertices]; + mGeneratedMesh->mNormals[pos] = nOut; + } - if (haveNormal) - { - if (mGeneratedMesh->HasNormals() ) - mGeneratedMesh->mNormals = new aiVector3D[mGeneratedMesh->mNumVertices]; - mGeneratedMesh->mNormals[pos] = nOut; - } + if (haveColor) { + if (nullptr == mGeneratedMesh->mColors[0]) + mGeneratedMesh->mColors[0] = new aiColor4D[mGeneratedMesh->mNumVertices]; + mGeneratedMesh->mColors[0][pos] = cOut; + } - if (haveColor) - { - if (mGeneratedMesh->HasVertexColors( 0 ) ) - mGeneratedMesh->mColors[0] = new aiColor4D[mGeneratedMesh->mNumVertices]; - mGeneratedMesh->mColors[0][pos] = cOut; + if (haveTextureCoords) { + if (nullptr == mGeneratedMesh->mTextureCoords[0]) { + mGeneratedMesh->mNumUVComponents[0] = 2; + mGeneratedMesh->mTextureCoords[0] = new aiVector3D[mGeneratedMesh->mNumVertices]; + } + mGeneratedMesh->mTextureCoords[0][pos] = tOut; + } } - - if (haveTextureCoords) - { - if (mGeneratedMesh->HasTextureCoords(0)) - { - mGeneratedMesh->mNumUVComponents[0] = 2; - mGeneratedMesh->mTextureCoords[0] = new aiVector3D[mGeneratedMesh->mNumVertices]; - } - mGeneratedMesh->mTextureCoords[0][pos] = tOut; - } - } } diff --git a/test/unit/utPLYImportExport.cpp b/test/unit/utPLYImportExport.cpp index 1ef8b6803..57a42ec15 100644 --- a/test/unit/utPLYImportExport.cpp +++ b/test/unit/utPLYImportExport.cpp @@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include "AbstractImportExportBase.h" using namespace ::Assimp; @@ -51,6 +52,11 @@ public: virtual bool importerTest() { Assimp::Importer importer; const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/PLY/cube.ply", 0 ); + EXPECT_EQ( 1u, scene->mNumMeshes ); + EXPECT_NE( nullptr, scene->mMeshes[0] ); + EXPECT_EQ( 8u, scene->mMeshes[0]->mNumVertices ); + EXPECT_EQ( 6u, scene->mMeshes[0]->mNumFaces ); + return nullptr != scene; } From 267d3f41e83027ba9bcc041b7cf89be9f0ca0409 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 3 Aug 2017 14:58:39 +0200 Subject: [PATCH 043/305] ply-loader: add brackets. --- test/unit/utPLYImportExport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/utPLYImportExport.cpp b/test/unit/utPLYImportExport.cpp index 57a42ec15..82cc54bdb 100644 --- a/test/unit/utPLYImportExport.cpp +++ b/test/unit/utPLYImportExport.cpp @@ -57,7 +57,7 @@ public: EXPECT_EQ( 8u, scene->mMeshes[0]->mNumVertices ); EXPECT_EQ( 6u, scene->mMeshes[0]->mNumFaces ); - return nullptr != scene; + return (nullptr != scene); } #ifndef ASSIMP_BUILD_NO_EXPORT From c3cd7349d0afc3aede84a90f0f45f7993f029329 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 5 Aug 2017 18:46:34 +0200 Subject: [PATCH 044/305] unzip: latest greatest. --- contrib/unzip/ioapi.c | 34 ++++++++++------------ contrib/unzip/ioapi.h | 16 +++++------ contrib/unzip/unzip.c | 36 ++++++++++------------- contrib/unzip/unzip.h | 66 ++++++++++++++++++++----------------------- 4 files changed, 69 insertions(+), 83 deletions(-) diff --git a/contrib/unzip/ioapi.c b/contrib/unzip/ioapi.c index b7200df75..f1bee23e6 100644 --- a/contrib/unzip/ioapi.c +++ b/contrib/unzip/ioapi.c @@ -10,11 +10,7 @@ #include #include -# ifdef ASSIMP_BUILD_NO_OWN_ZLIB -# include -# else -# include "../zlib/zlib.h" -# endif +#include "zlib.h" #include "ioapi.h" @@ -33,40 +29,40 @@ #define SEEK_SET 0 #endif -voidpf ZCALLBACK fopen_file_func ( +voidpf ZCALLBACK fopen_file_func OF(( voidpf opaque, const char* filename, - int mode); + int mode)); -uLong ZCALLBACK fread_file_func ( +uLong ZCALLBACK fread_file_func OF(( voidpf opaque, voidpf stream, void* buf, - uLong size); + uLong size)); -uLong ZCALLBACK fwrite_file_func ( +uLong ZCALLBACK fwrite_file_func OF(( voidpf opaque, voidpf stream, const void* buf, - uLong size); + uLong size)); -long ZCALLBACK ftell_file_func ( +long ZCALLBACK ftell_file_func OF(( voidpf opaque, - voidpf stream); + voidpf stream)); -long ZCALLBACK fseek_file_func ( +long ZCALLBACK fseek_file_func OF(( voidpf opaque, voidpf stream, uLong offset, - int origin); + int origin)); -int ZCALLBACK fclose_file_func ( +int ZCALLBACK fclose_file_func OF(( voidpf opaque, - voidpf stream); + voidpf stream)); -int ZCALLBACK ferror_file_func ( +int ZCALLBACK ferror_file_func OF(( voidpf opaque, - voidpf stream); + voidpf stream)); voidpf ZCALLBACK fopen_file_func (opaque, filename, mode) diff --git a/contrib/unzip/ioapi.h b/contrib/unzip/ioapi.h index 06fdd15e9..7d457baab 100644 --- a/contrib/unzip/ioapi.h +++ b/contrib/unzip/ioapi.h @@ -35,13 +35,13 @@ extern "C" { #endif -typedef voidpf (ZCALLBACK *open_file_func) (voidpf opaque, const char* filename, int mode); -typedef uLong (ZCALLBACK *read_file_func) (voidpf opaque, voidpf stream, void* buf, uLong size); -typedef uLong (ZCALLBACK *write_file_func)(voidpf opaque, voidpf stream, const void* buf, uLong size); -typedef long (ZCALLBACK *tell_file_func) (voidpf opaque, voidpf stream); -typedef long (ZCALLBACK *seek_file_func) (voidpf opaque, voidpf stream, uLong offset, int origin); -typedef int (ZCALLBACK *close_file_func) (voidpf opaque, voidpf stream); -typedef int (ZCALLBACK *testerror_file_func) (voidpf opaque, voidpf stream); +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); typedef struct zlib_filefunc_def_s { @@ -57,7 +57,7 @@ typedef struct zlib_filefunc_def_s -void fill_fopen_filefunc (zlib_filefunc_def* pzlib_filefunc_def); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); #define ZREAD(filefunc,filestream,buf,size) ((*((filefunc).zread_file))((filefunc).opaque,filestream,buf,size)) #define ZWRITE(filefunc,filestream,buf,size) ((*((filefunc).zwrite_file))((filefunc).opaque,filestream,buf,size)) diff --git a/contrib/unzip/unzip.c b/contrib/unzip/unzip.c index af00f7d6f..9ad4766d8 100644 --- a/contrib/unzip/unzip.c +++ b/contrib/unzip/unzip.c @@ -38,7 +38,8 @@ woven in by Terry Thorsen 1/2003. #include #include #include -#include "./unzip.h" +#include "zlib.h" +#include "unzip.h" #ifdef STDC # include @@ -162,10 +163,10 @@ typedef struct */ -local int unzlocal_getByte ( +local int unzlocal_getByte OF(( const zlib_filefunc_def* pzlib_filefunc_def, voidpf filestream, - int *pi); + int *pi)); local int unzlocal_getByte(pzlib_filefunc_def,filestream,pi) const zlib_filefunc_def* pzlib_filefunc_def; @@ -192,10 +193,10 @@ local int unzlocal_getByte(pzlib_filefunc_def,filestream,pi) /* =========================================================================== Reads a long in LSB order from the given gz_stream. Sets */ -local int unzlocal_getShort ( +local int unzlocal_getShort OF(( const zlib_filefunc_def* pzlib_filefunc_def, voidpf filestream, - uLong *pX); + uLong *pX)); local int unzlocal_getShort (pzlib_filefunc_def,filestream,pX) const zlib_filefunc_def* pzlib_filefunc_def; @@ -220,10 +221,10 @@ local int unzlocal_getShort (pzlib_filefunc_def,filestream,pX) return err; } -local int unzlocal_getLong ( +local int unzlocal_getLong OF(( const zlib_filefunc_def* pzlib_filefunc_def, voidpf filestream, - uLong *pX); + uLong *pX)); local int unzlocal_getLong (pzlib_filefunc_def,filestream,pX) const zlib_filefunc_def* pzlib_filefunc_def; @@ -323,9 +324,9 @@ extern int ZEXPORT unzStringFileNameCompare (fileName1,fileName2,iCaseSensitivit Locate the Central directory of a zipfile (at the end, just before the global comment) */ -local uLong unzlocal_SearchCentralDir ( +local uLong unzlocal_SearchCentralDir OF(( const zlib_filefunc_def* pzlib_filefunc_def, - voidpf filestream); + voidpf filestream)); local uLong unzlocal_SearchCentralDir(pzlib_filefunc_def,filestream) const zlib_filefunc_def* pzlib_filefunc_def; @@ -562,7 +563,7 @@ local void unzlocal_DosDateToTmuDate (ulDosDate, ptm) /* Get Info about the current file in the zipfile, with internal only info */ -local int unzlocal_GetCurrentFileInfoInternal (unzFile file, +local int unzlocal_GetCurrentFileInfoInternal OF((unzFile file, unz_file_info *pfile_info, unz_file_info_internal *pfile_info_internal, @@ -571,7 +572,7 @@ local int unzlocal_GetCurrentFileInfoInternal (unzFile file, void *extraField, uLong extraFieldBufferSize, char *szComment, - uLong commentBufferSize); + uLong commentBufferSize)); local int unzlocal_GetCurrentFileInfoInternal (file, pfile_info, @@ -607,12 +608,10 @@ local int unzlocal_GetCurrentFileInfoInternal (file, /* we check the magic */ if (err==UNZ_OK) - { if (unzlocal_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) err=UNZ_ERRNO; else if (uMagic!=0x02014b50) err=UNZ_BADZIPFILE; - } if (unzlocal_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) err=UNZ_ERRNO; @@ -689,12 +688,10 @@ local int unzlocal_GetCurrentFileInfoInternal (file, uSizeRead = extraFieldBufferSize; if (lSeek!=0) - { if (ZSEEK(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) lSeek=0; else err=UNZ_ERRNO; - } if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) if (ZREAD(s->z_filefunc, s->filestream,extraField,uSizeRead)!=uSizeRead) err=UNZ_ERRNO; @@ -716,12 +713,10 @@ local int unzlocal_GetCurrentFileInfoInternal (file, uSizeRead = commentBufferSize; if (lSeek!=0) - { if (ZSEEK(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) lSeek=0; else err=UNZ_ERRNO; - } if ((file_info.size_file_comment>0) && (commentBufferSize>0)) if (ZREAD(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) err=UNZ_ERRNO; @@ -982,13 +977,11 @@ local int unzlocal_CheckCurrentFileCoherencyHeader (s,piSizeVar, if (err==UNZ_OK) - { if (unzlocal_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) err=UNZ_ERRNO; else if (uMagic!=0x04034b50) err=UNZ_BADZIPFILE; - } - + if (unzlocal_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) err=UNZ_ERRNO; /* @@ -1246,7 +1239,7 @@ extern int ZEXPORT unzReadCurrentFile (file, buf, len) return UNZ_PARAMERROR; - if (pfile_in_zip_read_info->read_buffer == NULL) + if ((pfile_in_zip_read_info->read_buffer == NULL)) return UNZ_END_OF_LIST_OF_FILE; if (len==0) return 0; @@ -1541,6 +1534,7 @@ extern int ZEXPORT unzGetGlobalComment (file, szComment, uSizeBuf) char *szComment; uLong uSizeBuf; { + int err=UNZ_OK; unz_s* s; uLong uReadThis ; if (file==NULL) diff --git a/contrib/unzip/unzip.h b/contrib/unzip/unzip.h index e3b7f24ee..b247937c8 100644 --- a/contrib/unzip/unzip.h +++ b/contrib/unzip/unzip.h @@ -50,11 +50,7 @@ extern "C" { #endif #ifndef _ZLIB_H -# ifdef ASSIMP_BUILD_NO_OWN_ZLIB -# include -# else -# include "../zlib/zlib.h" -# endif +#include "zlib.h" #endif #ifndef _ZLIBIOAPI_H @@ -123,9 +119,9 @@ typedef struct unz_file_info_s tm_unz tmu_date; } unz_file_info; -extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, const char* fileName2, - int iCaseSensitivity); + int iCaseSensitivity)); /* Compare two filename (fileName1,fileName2). If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) @@ -136,7 +132,7 @@ extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, */ -extern unzFile ZEXPORT unzOpen (const char *path); +extern unzFile ZEXPORT unzOpen OF((const char *path)); /* Open a Zip file. path contain the full pathname (by example, on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer @@ -147,31 +143,31 @@ extern unzFile ZEXPORT unzOpen (const char *path); of this unzip package. */ -extern unzFile ZEXPORT unzOpen2 (const char *path, - zlib_filefunc_def* pzlib_filefunc_def); +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); /* Open a Zip file, like unzOpen, but provide a set of file low level API for read/write the zip file (see ioapi.h) */ -extern int ZEXPORT unzClose (unzFile file); +extern int ZEXPORT unzClose OF((unzFile file)); /* Close a ZipFile opened with unzipOpen. If there is files inside the .Zip opened with unzOpenCurrentFile (see later), these files MUST be closed with unzipCloseCurrentFile before call unzipClose. return UNZ_OK if there is no problem. */ -extern int ZEXPORT unzGetGlobalInfo (unzFile file, - unz_global_info *pglobal_info); +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); /* Write info about the ZipFile in the *pglobal_info structure. No preparation of the structure is needed return UNZ_OK if there is no problem. */ -extern int ZEXPORT unzGetGlobalComment (unzFile file, +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, char *szComment, - uLong uSizeBuf); + uLong uSizeBuf)); /* Get the global comment string of the ZipFile, in the szComment buffer. uSizeBuf is the size of the szComment buffer. @@ -182,22 +178,22 @@ extern int ZEXPORT unzGetGlobalComment (unzFile file, /***************************************************************************/ /* Unzip package allow you browse the directory of the zipfile */ -extern int ZEXPORT unzGoToFirstFile (unzFile file); +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); /* Set the current file of the zipfile to the first file. return UNZ_OK if there is no problem */ -extern int ZEXPORT unzGoToNextFile (unzFile file); +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); /* Set the current file of the zipfile to the next file. return UNZ_OK if there is no problem return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. */ -extern int ZEXPORT unzLocateFile (unzFile file, +extern int ZEXPORT unzLocateFile OF((unzFile file, const char *szFileName, - int iCaseSensitivity); + int iCaseSensitivity)); /* Try locate the file szFileName in the zipfile. For the iCaseSensitivity signification, see unzStringFileNameCompare @@ -227,14 +223,14 @@ extern int ZEXPORT unzGoToFilePos( /* ****************************************** */ -extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, unz_file_info *pfile_info, char *szFileName, uLong fileNameBufferSize, void *extraField, uLong extraFieldBufferSize, char *szComment, - uLong commentBufferSize); + uLong commentBufferSize)); /* Get Info about the current file if pfile_info!=NULL, the *pfile_info structure will contain somes info about @@ -253,24 +249,24 @@ extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, from it, and close it (you can close it before reading all the file) */ -extern int ZEXPORT unzOpenCurrentFile (unzFile file); +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); /* Open for reading data the current file in the zipfile. If there is no error, the return value is UNZ_OK. */ -extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, - const char* password); +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); /* Open for reading data the current file in the zipfile. password is a crypting password If there is no error, the return value is UNZ_OK. */ -extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, int* method, int* level, - int raw); + int raw)); /* Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) if raw==1 @@ -280,11 +276,11 @@ extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, but you CANNOT set method parameter as NULL */ -extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, int* method, int* level, int raw, - const char* password); + const char* password)); /* Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) if raw==1 @@ -295,15 +291,15 @@ extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, */ -extern int ZEXPORT unzCloseCurrentFile (unzFile file); +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); /* Close the file in zip opened with unzOpenCurrentFile Return UNZ_CRCERROR if all the file was read but the CRC is not good */ -extern int ZEXPORT unzReadCurrentFile (unzFile file, +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, voidp buf, - unsigned len); + unsigned len)); /* Read bytes from the current file (opened by unzOpenCurrentFile) buf contain buffer where data must be copied @@ -315,19 +311,19 @@ extern int ZEXPORT unzReadCurrentFile (unzFile file, (UNZ_ERRNO for IO error, or zLib error for uncompress error) */ -extern z_off_t ZEXPORT unztell (unzFile file); +extern z_off_t ZEXPORT unztell OF((unzFile file)); /* Give the current position in uncompressed data */ -extern int ZEXPORT unzeof (unzFile file); +extern int ZEXPORT unzeof OF((unzFile file)); /* return 1 if the end of file was reached, 0 elsewhere */ -extern int ZEXPORT unzGetLocalExtrafield (unzFile file, +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, voidp buf, - unsigned len); + unsigned len)); /* Read extra field from the current file (opened by unzOpenCurrentFile) This is the local-header version of the extra field (sometimes, there is From 7fd9c3dc980a2280ba2f781f3022730b43bc0e6b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 5 Aug 2017 18:54:17 +0200 Subject: [PATCH 045/305] =?UTF-8?q?=C2=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- code/FBXDocument.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/code/FBXDocument.cpp b/code/FBXDocument.cpp index 9b6e737a6..b19ce4cd0 100644 --- a/code/FBXDocument.cpp +++ b/code/FBXDocument.cpp @@ -292,11 +292,10 @@ Document::~Document() } // ------------------------------------------------------------------------------------------------ -static const int LowerSupportedVersion = 7100; -static const int UpperSupportedVersion = 7400; +static const unsigned int LowerSupportedVersion = 7100; +static const unsigned int UpperSupportedVersion = 7400; -void Document::ReadHeader() -{ +void Document::ReadHeader() { // Read ID objects from "Objects" section const Scope& sc = parser.GetRootScope(); const Element* const ehead = sc["FBXHeaderExtension"]; From 7a4a32625cd788ee4326f00f368eeb7f5c544885 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 9 Aug 2017 10:45:54 -0400 Subject: [PATCH 046/305] Ensure gltf asset version is printed as \d.0 --- code/glTF2Asset.h | 6 +++--- code/glTF2Asset.inl | 13 +++++++------ code/glTF2AssetWriter.inl | 4 ++-- code/glTF2Exporter.cpp | 2 +- code/glTFAsset.h | 4 ++-- code/glTFAsset.inl | 8 ++++---- code/glTFAssetWriter.inl | 4 ++-- code/glTFExporter.cpp | 2 +- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index bbef4413e..100ed9d58 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -177,7 +177,7 @@ namespace glTF2 struct GLB_Header { uint8_t magic[4]; //!< Magic number: "glTF" - uint32_t version; //!< Version number + float_t version; //!< Version number uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes uint32_t sceneLength; //!< Length, in bytes, of the glTF scene uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) @@ -1074,13 +1074,13 @@ namespace glTF2 std::string version; //!< Specifies the target rendering API (default: "1.0.3") } profile; //!< Specifies the target rendering API and version, e.g., WebGL 1.0.3. (default: {}) - int version; //!< The glTF format version + float version; //!< The glTF format version void Read(Document& doc); AssetMetadata() : premultipliedAlpha(false) - , version(0) + , version(0.) { } }; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 33598e243..915e4e11c 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -1229,7 +1229,7 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { // read the version, etc. - int statedVersion = 0; + float statedVersion = 0.; if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); @@ -1244,14 +1244,15 @@ inline void AssetMetadata::Read(Document& doc) } version = std::max(statedVersion, version); - if (version == 0) { - // if missing version, we'll assume version 1... - version = 1; + + if (version == 0.) { + // if missing version, we'll assume version 1.0... + version = 1.; } - if (version != 1) { + if (version != 1.) { char msg[128]; - ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %d", version); + ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.1f", version); throw DeadlyImportError(msg); } } diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 6e1865bbf..250f425c8 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -574,7 +574,7 @@ namespace glTF2 { GLB_Header header; memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); - header.version = 2; + header.version = 2.; AI_SWAP4(header.version); header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); @@ -600,7 +600,7 @@ namespace glTF2 { asset.SetObject(); { char versionChar[10]; - ai_snprintf(versionChar, sizeof(versionChar), "%d", mAsset.asset.version); + ai_snprintf(versionChar, sizeof(versionChar), "%.1f", mAsset.asset.version); asset.AddMember("version", Value(versionChar, mAl).Move(), mAl); asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index b6a2efdd0..8f968f952 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -845,7 +845,7 @@ void glTF2Exporter::ExportScene() void glTF2Exporter::ExportMetadata() { AssetMetadata& asset = mAsset->asset; - asset.version = 2; + asset.version = 2.; char buffer[256]; ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", diff --git a/code/glTFAsset.h b/code/glTFAsset.h index 41d0dfd06..ff9376556 100644 --- a/code/glTFAsset.h +++ b/code/glTFAsset.h @@ -177,7 +177,7 @@ namespace glTF struct GLB_Header { uint8_t magic[4]; //!< Magic number: "glTF" - uint32_t version; //!< Version number (always 1 as of the last update) + float_t version; //!< Version number (always 1 as of the last update) uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes uint32_t sceneLength; //!< Length, in bytes, of the glTF scene uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) @@ -1058,7 +1058,7 @@ namespace glTF std::string version; //!< Specifies the target rendering API (default: "1.0.3") } profile; //!< Specifies the target rendering API and version, e.g., WebGL 1.0.3. (default: {}) - int version; //!< The glTF format version (should be 1) + float version; //!< The glTF format version (should be 1.0) void Read(Document& doc); diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index fe29dde70..48aeaf855 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -1228,7 +1228,7 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { // read the version, etc. - int statedVersion = 0; + float statedVersion = 0.; if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); @@ -1243,14 +1243,14 @@ inline void AssetMetadata::Read(Document& doc) } version = std::max(statedVersion, version); - if (version == 0) { + if (version == 0.) { // if missing version, we'll assume version 1... version = 1; } - if (version != 1) { + if (version != 1.) { char msg[128]; - ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %d", version); + ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.0f", version); throw DeadlyImportError(msg); } } diff --git a/code/glTFAssetWriter.inl b/code/glTFAssetWriter.inl index d3a553c27..95fc87791 100644 --- a/code/glTFAssetWriter.inl +++ b/code/glTFAssetWriter.inl @@ -582,7 +582,7 @@ namespace glTF { GLB_Header header; memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); - header.version = 1; + header.version = 1.; AI_SWAP4(header.version); header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); @@ -608,7 +608,7 @@ namespace glTF { asset.SetObject(); { char versionChar[10]; - ai_snprintf(versionChar, sizeof(versionChar), "%d", mAsset.asset.version); + ai_snprintf(versionChar, sizeof(versionChar), "%.0f", mAsset.asset.version); asset.AddMember("version", Value(versionChar, mAl).Move(), mAl); asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index aeb5069aa..0de843d75 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -834,7 +834,7 @@ void glTFExporter::ExportScene() void glTFExporter::ExportMetadata() { glTF::AssetMetadata& asset = mAsset->asset; - asset.version = 1; + asset.version = 1.; char buffer[256]; ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", From 83bfa61f8d213bcf2b860d5bbb18b76834800ff9 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 9 Aug 2017 11:07:08 -0400 Subject: [PATCH 047/305] version in glb header is stored as uint32_t Validator complains about `1` not being a valid version, however. --- code/glTF2Asset.h | 2 +- code/glTFAsset.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 100ed9d58..416c9e7bc 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -177,7 +177,7 @@ namespace glTF2 struct GLB_Header { uint8_t magic[4]; //!< Magic number: "glTF" - float_t version; //!< Version number + uint32_t version; //!< Version number uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes uint32_t sceneLength; //!< Length, in bytes, of the glTF scene uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) diff --git a/code/glTFAsset.h b/code/glTFAsset.h index ff9376556..30d0bc056 100644 --- a/code/glTFAsset.h +++ b/code/glTFAsset.h @@ -177,7 +177,7 @@ namespace glTF struct GLB_Header { uint8_t magic[4]; //!< Magic number: "glTF" - float_t version; //!< Version number (always 1 as of the last update) + uint32_t version; //!< Version number (always 1 as of the last update) uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes uint32_t sceneLength; //!< Length, in bytes, of the glTF scene uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) From 5b3b80cbc2fa252ebbe6f95c189cada366a2d63f Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 9 Aug 2017 11:36:22 -0400 Subject: [PATCH 048/305] Formatting --- code/glTF2Asset.h | 2 +- code/glTF2Asset.inl | 8 ++++---- code/glTF2AssetWriter.inl | 2 +- code/glTF2Exporter.cpp | 6 +++--- code/glTFAsset.inl | 6 +++--- code/glTFAssetWriter.inl | 2 +- code/glTFExporter.cpp | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 416c9e7bc..52bc18268 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -1080,7 +1080,7 @@ namespace glTF2 AssetMetadata() : premultipliedAlpha(false) - , version(0.) + , version(0) { } }; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 915e4e11c..44a1cce00 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -1229,7 +1229,7 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { // read the version, etc. - float statedVersion = 0.; + float statedVersion = 0; if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); @@ -1245,12 +1245,12 @@ inline void AssetMetadata::Read(Document& doc) version = std::max(statedVersion, version); - if (version == 0.) { + if (version == 0) { // if missing version, we'll assume version 1.0... - version = 1.; + version = 1; } - if (version != 1.) { + if (version != 1) { char msg[128]; ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.1f", version); throw DeadlyImportError(msg); diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 250f425c8..ed8ee80ea 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -574,7 +574,7 @@ namespace glTF2 { GLB_Header header; memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); - header.version = 2.; + header.version = 2; AI_SWAP4(header.version); header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 8f968f952..11694a89d 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -466,8 +466,8 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, RefmWeights[idx_weights].mWeight; // A vertex can only have at most four joint weights. Ignore all others. - if (jointsPerVertex[vertexId] > 3) { - continue; + if (jointsPerVertex[vertexId] > 3) { + continue; } vertexJointData[vertexId][jointsPerVertex[vertexId]] = jointNamesIndex; @@ -845,7 +845,7 @@ void glTF2Exporter::ExportScene() void glTF2Exporter::ExportMetadata() { AssetMetadata& asset = mAsset->asset; - asset.version = 2.; + asset.version = 2; char buffer[256]; ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index 48aeaf855..91d36c59b 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -1228,7 +1228,7 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { // read the version, etc. - float statedVersion = 0.; + float statedVersion = 0; if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); @@ -1243,12 +1243,12 @@ inline void AssetMetadata::Read(Document& doc) } version = std::max(statedVersion, version); - if (version == 0.) { + if (version == 0) { // if missing version, we'll assume version 1... version = 1; } - if (version != 1.) { + if (version != 1) { char msg[128]; ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.0f", version); throw DeadlyImportError(msg); diff --git a/code/glTFAssetWriter.inl b/code/glTFAssetWriter.inl index 95fc87791..166b84fd1 100644 --- a/code/glTFAssetWriter.inl +++ b/code/glTFAssetWriter.inl @@ -582,7 +582,7 @@ namespace glTF { GLB_Header header; memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); - header.version = 1.; + header.version = 1; AI_SWAP4(header.version); header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index 0de843d75..f3fa5e526 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -469,8 +469,8 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, RefmWeights[idx_weights].mWeight; // A vertex can only have at most four joint weights. Ignore all others. - if (jointsPerVertex[vertexId] > 3) { - continue; + if (jointsPerVertex[vertexId] > 3) { + continue; } vertexJointData[vertexId][jointsPerVertex[vertexId]] = jointNamesIndex; @@ -834,7 +834,7 @@ void glTFExporter::ExportScene() void glTFExporter::ExportMetadata() { glTF::AssetMetadata& asset = mAsset->asset; - asset.version = 1.; + asset.version = 1; char buffer[256]; ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", From 847e0291a03a15be41f5f2329555777b5fe64bcc Mon Sep 17 00:00:00 2001 From: Jeremy Abel Date: Sat, 12 Aug 2017 00:13:24 -0400 Subject: [PATCH 049/305] changed a few leftover asserts to ai_assert --- code/C4DImporter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/C4DImporter.cpp b/code/C4DImporter.cpp index 252766ca6..39514f6b8 100644 --- a/code/C4DImporter.cpp +++ b/code/C4DImporter.cpp @@ -393,7 +393,7 @@ void C4DImporter::RecurseHierarchy(BaseObject* object, aiNode* parent) // ------------------------------------------------------------------------------------------------ aiMesh* C4DImporter::ReadMesh(BaseObject* object) { - assert(object != NULL && object->GetType() == Opolygon); + ai_assert(object != NULL && object->GetType() == Opolygon); // based on Melange sample code PolygonObject* const polyObject = dynamic_cast(object); @@ -635,7 +635,7 @@ unsigned int C4DImporter::ResolveMaterial(PolygonObject* obj) TextureTag& ttag = dynamic_cast(*tag); BaseMaterial* const mat = ttag.GetMaterial(); - assert(mat != NULL); + ai_assert(mat != NULL); const MaterialMap::const_iterator it = material_mapping.find(mat); if(it == material_mapping.end()) { From b1313b04b8274106a3affd42db7c72d0bce8e5ff Mon Sep 17 00:00:00 2001 From: Jeremy Abel Date: Sat, 12 Aug 2017 00:16:00 -0400 Subject: [PATCH 050/305] should be map, not set --- code/C4DImporter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/C4DImporter.h b/code/C4DImporter.h index 3ba2a17c6..4dea82161 100644 --- a/code/C4DImporter.h +++ b/code/C4DImporter.h @@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "BaseImporter.h" #include "LogAux.h" -#include +#include struct aiNode; struct aiMesh; struct aiMaterial; From 7861c359ba6d9ec1a7dbd6d93e118e6faa7f4632 Mon Sep 17 00:00:00 2001 From: Jeremy Abel Date: Sat, 12 Aug 2017 15:06:36 -0400 Subject: [PATCH 051/305] Added wiki link to C4D file format --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 4eb884c63..d09208203 100644 --- a/Readme.md +++ b/Readme.md @@ -89,7 +89,7 @@ __Importers__: Additionally, some formats are supported by dependency on non-free code or external SDKs (not built by default): -- C4D (https://github.com/acgessler/assimp-cinema4d) +- C4D (https://github.com/assimp/assimp/wiki/Cinema4D-&-Melange) __Exporters__: From dab09859944ef18f09e8ade9b16a65e3bf3f06f6 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Mon, 14 Aug 2017 22:20:26 +0200 Subject: [PATCH 052/305] assert: remove assert with more than one statement and use only ai_assert. --- code/ColladaExporter.h | 5 ++++- code/D3MFOpcPackage.cpp | 2 +- code/FBXMeshGeometry.cpp | 1 - code/FBXProperties.h | 23 ++++++++--------------- code/FBXTokenizer.h | 1 - code/HMPLoader.cpp | 3 ++- code/IFCBoolean.cpp | 1 - code/IFCCurve.cpp | 2 -- code/IFCOpenings.cpp | 6 +++--- code/MDLLoader.cpp | 4 +++- code/SMDLoader.cpp | 4 +++- code/XFileExporter.h | 5 ++++- tools/assimp_qt_viewer/glview.cpp | 4 ++-- 13 files changed, 30 insertions(+), 31 deletions(-) diff --git a/code/ColladaExporter.h b/code/ColladaExporter.h index 36c012f2e..c78272726 100644 --- a/code/ColladaExporter.h +++ b/code/ColladaExporter.h @@ -128,7 +128,10 @@ protected: /// Enters a new xml element, which increases the indentation void PushTag() { startstr.append( " "); } /// Leaves an element, decreasing the indentation - void PopTag() { ai_assert( startstr.length() > 1); startstr.erase( startstr.length() - 2); } + void PopTag() { + ai_assert( startstr.length() > 1); + startstr.erase( startstr.length() - 2); + } /// Creates a mesh ID for the given mesh std::string GetMeshId( size_t pIndex) const { diff --git a/code/D3MFOpcPackage.cpp b/code/D3MFOpcPackage.cpp index ff6fddcd1..b6b1ba170 100644 --- a/code/D3MFOpcPackage.cpp +++ b/code/D3MFOpcPackage.cpp @@ -229,7 +229,7 @@ ZipFile::~ZipFile() { size_t ZipFile::Read(void* pvBuffer, size_t pSize, size_t pCount) { const size_t size = pSize * pCount; - assert(size <= m_Size); + ai_assert(size <= m_Size); std::memcpy(pvBuffer, m_Buffer, size); diff --git a/code/FBXMeshGeometry.cpp b/code/FBXMeshGeometry.cpp index 3a57cfde9..8d43a2436 100644 --- a/code/FBXMeshGeometry.cpp +++ b/code/FBXMeshGeometry.cpp @@ -243,7 +243,6 @@ const unsigned int* MeshGeometry::ToOutputVertexIndex( unsigned int in_index, un ai_assert( m_mapping_counts.size() == m_mapping_offsets.size() ); count = m_mapping_counts[ in_index ]; -// ai_assert( count != 0 ); ai_assert( m_mapping_offsets[ in_index ] + count <= m_mappings.size() ); return &m_mappings[ m_mapping_offsets[ in_index ] ]; diff --git a/code/FBXProperties.h b/code/FBXProperties.h index a89a858be..91b7c8128 100644 --- a/code/FBXProperties.h +++ b/code/FBXProperties.h @@ -55,14 +55,12 @@ namespace FBX { // Forward declarations class Element; - /** Represents a dynamic property. Type info added by deriving classes, * see #TypedProperty. Example: @verbatim P: "ShininessExponent", "double", "Number", "",0.5 @endvebatim - */ class Property { @@ -79,7 +77,6 @@ public: } }; - template class TypedProperty : public Property { @@ -138,38 +135,35 @@ private: // ------------------------------------------------------------------------------------------------ template -inline T PropertyGet(const PropertyTable& in, const std::string& name, - const T& defaultValue) -{ +inline +T PropertyGet(const PropertyTable& in, const std::string& name, const T& defaultValue) { const Property* const prop = in.Get(name); - if(!prop) { + if( nullptr == prop) { return defaultValue; } // strong typing, no need to be lenient const TypedProperty* const tprop = prop->As< TypedProperty >(); - if(!tprop) { + if( nullptr == tprop) { return defaultValue; } return tprop->Value(); } - // ------------------------------------------------------------------------------------------------ template -inline T PropertyGet(const PropertyTable& in, const std::string& name, - bool& result) -{ +inline +T PropertyGet(const PropertyTable& in, const std::string& name, bool& result) { const Property* const prop = in.Get(name); - if(!prop) { + if( nullptr == prop) { result = false; return T(); } // strong typing, no need to be lenient const TypedProperty* const tprop = prop->As< TypedProperty >(); - if(!tprop) { + if( nullptr == tprop) { result = false; return T(); } @@ -178,7 +172,6 @@ inline T PropertyGet(const PropertyTable& in, const std::string& name, return tprop->Value(); } - } //! FBX } //! Assimp diff --git a/code/FBXTokenizer.h b/code/FBXTokenizer.h index f8af0ca16..e00ff5b79 100644 --- a/code/FBXTokenizer.h +++ b/code/FBXTokenizer.h @@ -101,7 +101,6 @@ public: return std::string(begin(),end()); } -public: bool IsBinary() const { return column == BINARY_MARKER; } diff --git a/code/HMPLoader.cpp b/code/HMPLoader.cpp index a6a0262bf..2a669201e 100644 --- a/code/HMPLoader.cpp +++ b/code/HMPLoader.cpp @@ -448,7 +448,8 @@ void HMPImporter::CreateOutputFaceList(unsigned int width,unsigned int height) void HMPImporter::ReadFirstSkin(unsigned int iNumSkins, const unsigned char* szCursor, const unsigned char** szCursorOut) { - ai_assert(0 != iNumSkins && NULL != szCursor); + ai_assert( 0 != iNumSkins ); + ai_assert( nullptr != szCursor); // read the type of the skin ... // sometimes we need to skip 12 bytes here, I don't know why ... diff --git a/code/IFCBoolean.cpp b/code/IFCBoolean.cpp index 98b230392..8571a3c79 100644 --- a/code/IFCBoolean.cpp +++ b/code/IFCBoolean.cpp @@ -381,7 +381,6 @@ bool PointInPoly(const IfcVector3& p, const std::vector& boundary) IntersectsBoundaryProfile(p, p + IfcVector3(0.6, -0.6, 0.0), boundary, true, intersected_boundary, true); votes += intersected_boundary.size() % 2; -// ai_assert(votes == 3 || votes == 0); return votes > 1; } diff --git a/code/IFCCurve.cpp b/code/IFCCurve.cpp index ede1c87b8..176fe3c13 100644 --- a/code/IFCCurve.cpp +++ b/code/IFCCurve.cpp @@ -548,8 +548,6 @@ bool Curve :: InRange(IfcFloat u) const const ParamRange range = GetParametricRange(); if (IsClosed()) { return true; - //ai_assert(range.first != std::numeric_limits::infinity() && range.second != std::numeric_limits::infinity()); - //u = range.first + std::fmod(u-range.first,range.second-range.first); } const IfcFloat epsilon = 1e-5; return u - range.first > -epsilon && range.second - u > -epsilon; diff --git a/code/IFCOpenings.cpp b/code/IFCOpenings.cpp index f40b49fe4..45e0d1b90 100644 --- a/code/IFCOpenings.cpp +++ b/code/IFCOpenings.cpp @@ -1492,7 +1492,7 @@ bool TryAddOpenings_Poly2Tri(const std::vector& openings,const std: vmax -= vmin; // If this happens then the projection must have been wrong. - assert(vmax.Length()); + ai_assert(vmax.Length()); ClipperLib::ExPolygons clipped; ClipperLib::Polygons holes_union; @@ -1616,7 +1616,7 @@ bool TryAddOpenings_Poly2Tri(const std::vector& openings,const std: std::vector tmpvec; for(ClipperLib::Polygon& opening : holes_union) { - assert(ClipperLib::Orientation(opening)); + ai_assert(ClipperLib::Orientation(opening)); tmpvec.clear(); @@ -1705,7 +1705,7 @@ bool TryAddOpenings_Poly2Tri(const std::vector& openings,const std: static_cast( tri->GetPoint(i)->y ) ); - assert(v.x <= 1.0 && v.x >= 0.0 && v.y <= 1.0 && v.y >= 0.0); + ai_assert(v.x <= 1.0 && v.x >= 0.0 && v.y <= 1.0 && v.y >= 0.0); const IfcVector3 v3 = minv * IfcVector3(vmin.x + v.x * vmax.x, vmin.y + v.y * vmax.y,coord) ; curmesh.verts.push_back(v3); diff --git a/code/MDLLoader.cpp b/code/MDLLoader.cpp index ad63dd40b..2025d79b3 100644 --- a/code/MDLLoader.cpp +++ b/code/MDLLoader.cpp @@ -1132,7 +1132,9 @@ bool MDLImporter::ProcessFrames_3DGS_MDL7(const MDL::IntGroupInfo_MDL7& groupInf const unsigned char* szCurrent, const unsigned char** szCurrentOut) { - ai_assert(NULL != szCurrent && NULL != szCurrentOut); + ai_assert( nullptr != szCurrent ); + ai_assert( nullptr != szCurrentOut); + const MDL::Header_MDL7 *pcHeader = (const MDL::Header_MDL7*)mBuffer; // if we have no bones we can simply skip all frames, diff --git a/code/SMDLoader.cpp b/code/SMDLoader.cpp index 1e9f86bea..4502e8d7a 100644 --- a/code/SMDLoader.cpp +++ b/code/SMDLoader.cpp @@ -450,7 +450,9 @@ void SMDImporter::CreateOutputMeshes() // add bone child nodes void SMDImporter::AddBoneChildren(aiNode* pcNode, uint32_t iParent) { - ai_assert(NULL != pcNode && 0 == pcNode->mNumChildren && NULL == pcNode->mChildren); + ai_assert( NULL != pcNode ); + ai_assert( 0 == pcNode->mNumChildren ); + ai_assert( NULL == pcNode->mChildren); // first count ... for (unsigned int i = 0; i < asBones.size();++i) diff --git a/code/XFileExporter.h b/code/XFileExporter.h index 8d3478276..e45fd7983 100644 --- a/code/XFileExporter.h +++ b/code/XFileExporter.h @@ -93,7 +93,10 @@ protected: void PushTag() { startstr.append( " "); } /// Leaves an element, decreasing the indentation - void PopTag() { ai_assert( startstr.length() > 1); startstr.erase( startstr.length() - 2); } + void PopTag() { + ai_assert( startstr.length() > 1); + startstr.erase( startstr.length() - 2); + } public: /// Stringstream to write all output into diff --git a/tools/assimp_qt_viewer/glview.cpp b/tools/assimp_qt_viewer/glview.cpp index 1de8199fa..caf4ef320 100644 --- a/tools/assimp_qt_viewer/glview.cpp +++ b/tools/assimp_qt_viewer/glview.cpp @@ -142,8 +142,8 @@ void CGLView::Material_Apply(const aiMaterial* pMaterial) void CGLView::Matrix_NodeToRoot(const aiNode* pNode, aiMatrix4x4& pOutMatrix) { -const aiNode* node_cur; -std::list mat_list; + const aiNode* node_cur; + std::list mat_list; pOutMatrix = aiMatrix4x4(); // starting walk from current element to root From b26fea4cf5b908bba03e026b358a1778559c1dc8 Mon Sep 17 00:00:00 2001 From: Minmin Gong Date: Mon, 14 Aug 2017 21:51:59 -0700 Subject: [PATCH 053/305] Fix compiling problems under VS2017.3. 1. WordIterator's operator== and operator!= must take const WordIterator& as parameter type. 2. OpenMP doesn't work with new two phase lookups. Need to disable twoPhase. --- CMakeLists.txt | 6 ++++++ code/X3DImporter.cpp | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 676a48385..690351e8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,12 @@ find_package(OpenMP) if (OPENMP_FOUND) SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + + IF(MSVC) + IF(MSVC_VERSION GREATER 1910) + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:twoPhase-") + ENDIF() + ENDIF() endif() CONFIGURE_FILE( diff --git a/code/X3DImporter.cpp b/code/X3DImporter.cpp index 111773563..669f46f95 100644 --- a/code/X3DImporter.cpp +++ b/code/X3DImporter.cpp @@ -95,8 +95,8 @@ struct WordIterator: public std::iterator end_ = other.end_; return *this; } - bool operator==(WordIterator &other) const { return start_ == other.start_; } - bool operator!=(WordIterator &other) const { return start_ != other.start_; } + bool operator==(const WordIterator &other) const { return start_ == other.start_; } + bool operator!=(const WordIterator &other) const { return start_ != other.start_; } WordIterator &operator++() { start_ += strcspn(start_, whitespace); start_ += strspn(start_, whitespace); From e0fc412e570f6964feb56a96b6b6d218234b10a1 Mon Sep 17 00:00:00 2001 From: Arshia001 Date: Fri, 18 Aug 2017 17:40:07 +0430 Subject: [PATCH 054/305] Fix OBJ discarding all material names if the material library is missing --- code/ObjFileParser.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/code/ObjFileParser.cpp b/code/ObjFileParser.cpp index dfa77dba2..41677fce5 100644 --- a/code/ObjFileParser.cpp +++ b/code/ObjFileParser.cpp @@ -555,10 +555,15 @@ void ObjFileParser::getMaterialDesc() { // Search for material std::map::iterator it = m_pModel->m_MaterialMap.find(strName); if (it == m_pModel->m_MaterialMap.end()) { - // Not found, use default material - m_pModel->m_pCurrentMaterial = m_pModel->m_pDefaultMaterial; - DefaultLogger::get()->error("OBJ: failed to locate material " + strName + ", skipping"); - strName = m_pModel->m_pDefaultMaterial->MaterialName.C_Str(); + // Not found, so we don't know anything about the material except for its name. + // This may be the case if the material library is missing. We don't want to lose all + // materials if that happens, so create a new named material instead of discarding it + // completely. + DefaultLogger::get()->error("OBJ: failed to locate material " + strName + ", creating new material"); + m_pModel->m_pCurrentMaterial = new ObjFile::Material(); + m_pModel->m_pCurrentMaterial->MaterialName.Set(strName); + m_pModel->m_MaterialLib.push_back(strName); + m_pModel->m_MaterialMap[strName] = m_pModel->m_pCurrentMaterial; } else { // Found, using detected material m_pModel->m_pCurrentMaterial = (*it).second; From 91f6a9a721e56fa026ff62a6722702e74e9bf1fe Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 19 Aug 2017 22:09:30 +0200 Subject: [PATCH 055/305] Readme: add TriLib to the official list of supported ports. --- Readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Readme.md b/Readme.md index d09208203..5a59d0364 100644 --- a/Readme.md +++ b/Readme.md @@ -114,6 +114,7 @@ Take a look into the `INSTALL` file. Our build system is CMake, if you used CMak * [.NET](port/AssimpNET/Readme.md) * [Pascal](port/AssimpPascal/Readme.md) * [Javascript (Alpha)](https://github.com/makc/assimp2json) +* [Unity 3d Plugin] (https://www.assetstore.unity3d.com/en/#!/content/91777) ### Other tools ### [open3mod](https://github.com/acgessler/open3mod) is a powerful 3D model viewer based on Assimp's import and export abilities. From d0d45d1d22fad4ef4026ffca95b7b7916d8a83cc Mon Sep 17 00:00:00 2001 From: Lee Jeffery Date: Sun, 20 Aug 2017 19:07:09 +0100 Subject: [PATCH 056/305] Fix linking issue with irrXML --- contrib/irrXML/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/irrXML/CMakeLists.txt b/contrib/irrXML/CMakeLists.txt index 82ede3a04..980bd99a4 100644 --- a/contrib/irrXML/CMakeLists.txt +++ b/contrib/irrXML/CMakeLists.txt @@ -11,3 +11,7 @@ set( IrrXML_SRCS add_library(IrrXML STATIC ${IrrXML_SRCS}) set(IRRXML_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" CACHE INTERNAL "IrrXML_Include" ) set(IRRXML_LIBRARY "IrrXML" CACHE INTERNAL "IrrXML" ) + +install(TARGETS IrrXML + ARCHIVE DESTINATION ${ASSIMP_LIB_INSTALL_DIR} + COMPONENT ${LIBASSIMP_COMPONENT}) From 2e5e56c000567e7b7cf41d8357343772d818c01c Mon Sep 17 00:00:00 2001 From: Eduardo Costa Date: Mon, 21 Aug 2017 09:27:40 +0100 Subject: [PATCH 057/305] Re-enabling PACK_STRUCT for MDL files. When using this compiler: Apple LLVM version 7.3.0 (clang-703.0.31) Target: x86_64-apple-darwin15.4.0 Thread model: posix The MDL::Frame is aligned in 8-byte boundaries instead of 4. This generates this error when loading old Quake 1 MDL files: ERROR: Failed to load file: Invalid MDL file. The file is too small or contains invalid data (File: MDLLoader.cpp Line: 422) It seems this error was introduced by commit bd2af6591f20fd8777332d510056a037fbfa0df5 due to compilation warnings. --- code/MDLFileData.h | 56 +++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/code/MDLFileData.h b/code/MDLFileData.h index a691851be..2afea8a95 100644 --- a/code/MDLFileData.h +++ b/code/MDLFileData.h @@ -164,7 +164,7 @@ struct Header { //! Could be the total size of the file (and not a float) float size; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- @@ -223,7 +223,7 @@ struct Header_MDL7 { //! Size of the Frame_MDL7 data structure used in the file uint16_t frame_stc_size; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- @@ -242,7 +242,7 @@ struct Bone_MDL7 { //! Optional name of the bone char name[1 /* DUMMY SIZE */]; -} /* PACK_STRUCT */; +} PACK_STRUCT; #if (!defined AI_MDL7_BONE_STRUCT_SIZE__NAME_IS_20_CHARS) # define AI_MDL7_BONE_STRUCT_SIZE__NAME_IS_20_CHARS (16 + 20) @@ -290,7 +290,7 @@ struct Group_MDL7 { //! Number of frames int32_t numframes; -} /* PACK_STRUCT */; +} PACK_STRUCT; #define AI_MDL7_SKINTYPE_MIPFLAG 0x08 #define AI_MDL7_SKINTYPE_MATERIAL 0x10 @@ -312,7 +312,7 @@ struct Deformer_MDL7 { int32_t group_index; int32_t elements; int32_t deformerdata_size; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- @@ -324,7 +324,7 @@ struct DeformerElement_MDL7 { int32_t element_index; char element_name[AI_MDL7_MAX_BONENAMESIZE]; int32_t weights; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct DeformerWeight_MDL7 @@ -334,7 +334,7 @@ struct DeformerWeight_MDL7 { //! for deformer_typ==0 (==bones) index == vertex index int32_t index; float weight; -} /* PACK_STRUCT */; +} PACK_STRUCT; // don't know why this was in the original headers ... typedef int32_t MD7_MATERIAL_ASCDEFSIZE; @@ -345,7 +345,7 @@ typedef int32_t MD7_MATERIAL_ASCDEFSIZE; */ struct ColorValue_MDL7 { float r,g,b,a; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct Material_MDL7 @@ -366,7 +366,7 @@ struct Material_MDL7 { //! Phong power float Power; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct Skin @@ -388,7 +388,7 @@ struct Skin { //! Texture data uint8_t *data; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- @@ -399,7 +399,7 @@ struct Skin { struct Skin_MDL5 { int32_t size, width, height; uint8_t *data; -} /* PACK_STRUCT */; +} PACK_STRUCT; // maximum length of texture file name #if (!defined AI_MDL7_MAX_TEXNAMESIZE) @@ -416,7 +416,7 @@ struct Skin_MDL7 { int32_t width; int32_t height; char texture_name[AI_MDL7_MAX_TEXNAMESIZE]; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct RGB565 @@ -426,7 +426,7 @@ struct RGB565 { uint16_t r : 5; uint16_t g : 6; uint16_t b : 5; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct ARGB4 @@ -455,7 +455,7 @@ struct GroupSkin { //! Data of each image uint8_t **data; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct TexCoord @@ -470,7 +470,7 @@ struct TexCoord { //! Texture coordinate in the ty direction int32_t t; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct TexCoord_MDL3 @@ -482,7 +482,7 @@ struct TexCoord_MDL3 { //! position, vertically in range 0..skinheight-1 int16_t v; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct TexCoord_MDL7 @@ -494,7 +494,7 @@ struct TexCoord_MDL7 { //! position, vertically in range 0..1 float v; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct SkinSet_MDL7 @@ -510,7 +510,7 @@ struct SkinSet_MDL7 //! Material index int32_t material; // size 4 -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct Triangle @@ -523,7 +523,7 @@ struct Triangle //! Vertex indices int32_t vertex[3]; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct Triangle_MDL3 @@ -536,7 +536,7 @@ struct Triangle_MDL3 //! Index of 3 skin vertices in range 0..numskinverts uint16_t index_uv[3]; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct Triangle_MDL7 @@ -549,7 +549,7 @@ struct Triangle_MDL7 //! Two skinsets. The second will be used for multi-texturing SkinSet_MDL7 skinsets[2]; -} /* PACK_STRUCT */; +} PACK_STRUCT; #if (!defined AI_MDL7_TRIANGLE_STD_SIZE_ONE_UV) # define AI_MDL7_TRIANGLE_STD_SIZE_ONE_UV (6+sizeof(SkinSet_MDL7)-sizeof(uint32_t)) @@ -577,7 +577,7 @@ struct Vertex { uint8_t v[3]; uint8_t normalIndex; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- @@ -603,7 +603,7 @@ struct Vertex_MDL7 uint8_t norm162index; float norm[3]; }; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct BoneTransform_MDL7 @@ -620,7 +620,7 @@ struct BoneTransform_MDL7 //! I HATE 3DGS AND THE SILLY DEVELOPER WHO DESIGNED //! THIS STUPID FILE FORMAT! int8_t _unused_[2]; -} /* PACK_STRUCT */; +} PACK_STRUCT; #define AI_MDL7_MAX_FRAMENAMESIZE 16 @@ -654,7 +654,7 @@ struct SimpleFrame //! Vertex list of the frame Vertex *verts; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct Frame @@ -667,7 +667,7 @@ struct Frame //! Frame data SimpleFrame frame; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- @@ -684,7 +684,7 @@ struct SimpleFrame_MDLn_SP //! Vertex list of the frame Vertex_MDL4 *verts; -} /* PACK_STRUCT */; +} PACK_STRUCT; // ------------------------------------------------------------------------------------- /** \struct GroupFrame @@ -706,7 +706,7 @@ struct GroupFrame //! List of single frames SimpleFrame *frames; -} /* PACK_STRUCT */; +} PACK_STRUCT; #include "./../include/assimp/Compiler/poppack1.h" From 92beee99245dab00256f02d917845d0e2ce6b030 Mon Sep 17 00:00:00 2001 From: Tammo Hinrichs Date: Wed, 23 Aug 2017 17:52:53 +0200 Subject: [PATCH 058/305] Collada importer: Add support for line strip primitives --- code/ColladaParser.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/code/ColladaParser.cpp b/code/ColladaParser.cpp index dbbbdcb18..7b6592351 100644 --- a/code/ColladaParser.cpp +++ b/code/ColladaParser.cpp @@ -2231,7 +2231,7 @@ void ColladaParser::ReadIndexData( Mesh* pMesh) } #ifdef ASSIMP_BUILD_DEBUG - if (primType != Prim_TriFans && primType != Prim_TriStrips && + if (primType != Prim_TriFans && primType != Prim_TriStrips && primType != Prim_LineStrip && primType != Prim_Lines) { // this is ONLY to workaround a bug in SketchUp 15.3.331 where it writes the wrong 'count' when it writes out the 'lines'. ai_assert(actualPrimitives == numPrimitives); } @@ -2400,6 +2400,10 @@ size_t ColladaParser::ReadPrimitives( Mesh* pMesh, std::vector& pP size_t numberOfVertices = indices.size() / numOffsets; numPrimitives = numberOfVertices - 2; } + if (pPrimType == Prim_LineStrip) { + size_t numberOfVertices = indices.size() / numOffsets; + numPrimitives = numberOfVertices - 1; + } pMesh->mFaceSize.reserve( numPrimitives); pMesh->mFacePosIndices.reserve( indices.size() / numOffsets); @@ -2416,6 +2420,11 @@ size_t ColladaParser::ReadPrimitives( Mesh* pMesh, std::vector& pP for (size_t currentVertex = 0; currentVertex < numPoints; currentVertex++) CopyVertex(currentVertex, numOffsets, numPoints, perVertexOffset, pMesh, pPerIndexChannels, currentPrimitive, indices); break; + case Prim_LineStrip: + numPoints = 2; + for (size_t currentVertex = 0; currentVertex < numPoints; currentVertex++) + CopyVertex(currentVertex, numOffsets, 1, perVertexOffset, pMesh, pPerIndexChannels, currentPrimitive, indices); + break; case Prim_Triangles: numPoints = 3; for (size_t currentVertex = 0; currentVertex < numPoints; currentVertex++) From 37f5619149952ffdcfd01e7c34e6adb3b8bc6031 Mon Sep 17 00:00:00 2001 From: Mel Massadian Date: Wed, 23 Aug 2017 20:02:58 +0200 Subject: [PATCH 059/305] created a python3 version of the 3dviewer and fixed the / = float in py3 --- port/PyAssimp/pyassimp/helper.py | 54 +- port/PyAssimp/scripts/3d_viewer_py3.py | 1320 ++++++++++++++++++++++++ 2 files changed, 1353 insertions(+), 21 deletions(-) create mode 100755 port/PyAssimp/scripts/3d_viewer_py3.py diff --git a/port/PyAssimp/pyassimp/helper.py b/port/PyAssimp/pyassimp/helper.py index b281e94cf..4fa01d51d 100644 --- a/port/PyAssimp/pyassimp/helper.py +++ b/port/PyAssimp/pyassimp/helper.py @@ -9,6 +9,10 @@ import ctypes from ctypes import POINTER import operator +from distutils.sysconfig import get_python_lib +import re +import sys + try: import numpy except: numpy = None @@ -26,7 +30,15 @@ if os.name=='posix': additional_dirs.append('/usr/lib/x86_64-linux-gnu') additional_dirs.append('/usr/local/lib/') - # note - this won't catch libassimp.so.N.n, but + # check if running from anaconda. + if "conda" or "continuum" in sys.version.lower(): + cur_path = get_python_lib() + pattern = re.compile('.*\/lib\/') + conda_lib = pattern.match(cur_path).group() + logger.info("Adding Anaconda lib path:"+ conda_lib) + additional_dirs.append(conda_lib) + + # note - this won't catch libassimp.so.N.n, but # currently there's always a symlink called # libassimp.so in /usr/local/lib. ext_whitelist.append('.so') @@ -39,7 +51,7 @@ elif os.name=='nt': for dir_candidate in path_dirs: if 'assimp' in dir_candidate.lower(): additional_dirs.append(dir_candidate) - + #print(additional_dirs) def vec2tuple(x): """ Converts a VECTOR3D to a Tuple """ @@ -61,10 +73,10 @@ def transform(vector3, matrix4x4): m2[0]*x + m2[1]*y + m2[2]*z + m2[3], m3[0]*x + m3[1]*y + m3[2]*z + m3[3] ] - + def _inv(matrix4x4): m0,m1,m2,m3 = matrix4x4 - + det = m0[3]*m1[2]*m2[1]*m3[0] - m0[2]*m1[3]*m2[1]*m3[0] - \ m0[3]*m1[1]*m2[2]*m3[0] + m0[1]*m1[3]*m2[2]*m3[0] + \ m0[2]*m1[1]*m2[3]*m3[0] - m0[1]*m1[2]*m2[3]*m3[0] - \ @@ -77,7 +89,7 @@ def _inv(matrix4x4): m0[2]*m1[1]*m2[0]*m3[3] + m0[1]*m1[2]*m2[0]*m3[3] + \ m0[2]*m1[0]*m2[1]*m3[3] - m0[0]*m1[2]*m2[1]*m3[3] - \ m0[1]*m1[0]*m2[2]*m3[3] + m0[0]*m1[1]*m2[2]*m3[3] - + return[[( m1[2]*m2[3]*m3[1] - m1[3]*m2[2]*m3[1] + m1[3]*m2[1]*m3[2] - m1[1]*m2[3]*m3[2] - m1[2]*m2[1]*m3[3] + m1[1]*m2[2]*m3[3]) /det, ( m0[3]*m2[2]*m3[1] - m0[2]*m2[3]*m3[1] - m0[3]*m2[1]*m3[2] + m0[1]*m2[3]*m3[2] + m0[2]*m2[1]*m3[3] - m0[1]*m2[2]*m3[3]) /det, ( m0[2]*m1[3]*m3[1] - m0[3]*m1[2]*m3[1] + m0[3]*m1[1]*m3[2] - m0[1]*m1[3]*m3[2] - m0[2]*m1[1]*m3[3] + m0[1]*m1[2]*m3[3]) /det, @@ -94,7 +106,7 @@ def _inv(matrix4x4): ( m0[1]*m2[2]*m3[0] - m0[2]*m2[1]*m3[0] + m0[2]*m2[0]*m3[1] - m0[0]*m2[2]*m3[1] - m0[1]*m2[0]*m3[2] + m0[0]*m2[1]*m3[2]) /det, ( m0[2]*m1[1]*m3[0] - m0[1]*m1[2]*m3[0] - m0[2]*m1[0]*m3[1] + m0[0]*m1[2]*m3[1] + m0[1]*m1[0]*m3[2] - m0[0]*m1[1]*m3[2]) /det, ( m0[1]*m1[2]*m2[0] - m0[2]*m1[1]*m2[0] + m0[2]*m1[0]*m2[1] - m0[0]*m1[2]*m2[1] - m0[1]*m1[0]*m2[2] + m0[0]*m1[1]*m2[2]) /det]] - + def get_bounding_box(scene): bb_min = [1e10, 1e10, 1e10] # x,y,z bb_max = [-1e10, -1e10, -1e10] # x,y,z @@ -129,7 +141,7 @@ def get_bounding_box_for_node(node, bb_min, bb_max, transformation): t3[0]*T0[2] + t3[1]*T1[2] + t3[2]*T2[2] + t3[3]*T3[2], t3[0]*T0[3] + t3[1]*T1[3] + t3[2]*T2[3] + t3[3]*T3[3] ] ] - + for mesh in node.meshes: for v in mesh.vertices: v = transform(v, transformation) @@ -149,25 +161,25 @@ def get_bounding_box_for_node(node, bb_min, bb_max, transformation): def try_load_functions(library_path, dll): ''' Try to bind to aiImportFile and aiReleaseImport - + Arguments --------- library_path: path to current lib dll: ctypes handle to library - + Returns --------- If unsuccessful: None If successful: - Tuple containing (library_path, + Tuple containing (library_path, load from filename function, load from memory function, export to filename function, - release function, + release function, ctypes handle to assimp library) ''' - + try: load = dll.aiImportFile release = dll.aiReleaseImport @@ -176,7 +188,7 @@ def try_load_functions(library_path, dll): except AttributeError: #OK, this is a library, but it doesn't have the functions we need return None - + # library found! from .structs import Scene load.restype = POINTER(Scene) @@ -185,13 +197,13 @@ def try_load_functions(library_path, dll): def search_library(): ''' - Loads the assimp library. + Loads the assimp library. Throws exception AssimpError if no library_path is found - - Returns: tuple, (load from filename function, + + Returns: tuple, (load from filename function, load from memory function, export to filename function, - release function, + release function, dll) ''' #this path @@ -201,7 +213,7 @@ def search_library(): try: ctypes.windll.kernel32.SetErrorMode(0x8007) except AttributeError: - pass + pass candidates = [] # test every file @@ -209,7 +221,7 @@ def search_library(): if os.path.isdir(curfolder): for filename in os.listdir(curfolder): # our minimum requirement for candidates is that - # they should contain 'assimp' somewhere in + # they should contain 'assimp' somewhere in # their name if filename.lower().find('assimp')==-1 or\ os.path.splitext(filename)[-1].lower() not in ext_whitelist: @@ -248,10 +260,10 @@ def hasattr_silent(object, name): """ Calls hasttr() with the given parameters and preserves the legacy (pre-Python 3.2) functionality of silently catching exceptions. - + Returns the result of hasatter() or False if an exception was raised. """ - + try: return hasattr(object, name) except: diff --git a/port/PyAssimp/scripts/3d_viewer_py3.py b/port/PyAssimp/scripts/3d_viewer_py3.py new file mode 100755 index 000000000..4e4ecebe8 --- /dev/null +++ b/port/PyAssimp/scripts/3d_viewer_py3.py @@ -0,0 +1,1320 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +""" This program loads a model with PyASSIMP, and display it. + +Based on: +- pygame code from http://3dengine.org/Spectator_%28PyOpenGL%29 +- http://www.lighthouse3d.com/tutorials +- http://www.songho.ca/opengl/gl_transform.html +- http://code.activestate.com/recipes/325391/ +- ASSIMP's C++ SimpleOpenGL viewer + +Authors: Séverin Lemaignan, 2012-2016 +""" +import sys +import logging + +from functools import reduce + +logger = logging.getLogger("pyassimp") +gllogger = logging.getLogger("OpenGL") +gllogger.setLevel(logging.WARNING) +logging.basicConfig(level=logging.INFO) + +import OpenGL + +OpenGL.ERROR_CHECKING = False +OpenGL.ERROR_LOGGING = False +# OpenGL.ERROR_ON_COPY = True +# OpenGL.FULL_LOGGING = True +from OpenGL.GL import * +from OpenGL.arrays import vbo +from OpenGL.GL import shaders + +import pygame +import pygame.font +import pygame.image + +import math, random +from numpy import linalg + +import pyassimp +from pyassimp.postprocess import * +from pyassimp.helper import * +import transformations + +ROTATION_180_X = numpy.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]], dtype=numpy.float32) + +# rendering mode +BASE = "BASE" +COLORS = "COLORS" +SILHOUETTE = "SILHOUETTE" +HELPERS = "HELPERS" + +# Entities type +ENTITY = "entity" +CAMERA = "camera" +MESH = "mesh" + +FLAT_VERTEX_SHADER_120 = """ +#version 120 + +uniform mat4 u_viewProjectionMatrix; +uniform mat4 u_modelMatrix; + +uniform vec4 u_materialDiffuse; + +attribute vec3 a_vertex; + +varying vec4 v_color; + +void main(void) +{ + v_color = u_materialDiffuse; + gl_Position = u_viewProjectionMatrix * u_modelMatrix * vec4(a_vertex, 1.0); +} +""" + +FLAT_VERTEX_SHADER_130 = """ +#version 130 + +uniform mat4 u_viewProjectionMatrix; +uniform mat4 u_modelMatrix; + +uniform vec4 u_materialDiffuse; + +in vec3 a_vertex; + +out vec4 v_color; + +void main(void) +{ + v_color = u_materialDiffuse; + gl_Position = u_viewProjectionMatrix * u_modelMatrix * vec4(a_vertex, 1.0); +} +""" + +BASIC_VERTEX_SHADER_120 = """ +#version 120 + +uniform mat4 u_viewProjectionMatrix; +uniform mat4 u_modelMatrix; +uniform mat3 u_normalMatrix; +uniform vec3 u_lightPos; + +uniform vec4 u_materialDiffuse; + +attribute vec3 a_vertex; +attribute vec3 a_normal; + +varying vec4 v_color; + +void main(void) +{ + // Now the normal is in world space, as we pass the light in world space. + vec3 normal = u_normalMatrix * a_normal; + + float dist = distance(a_vertex, u_lightPos); + + // go to https://www.desmos.com/calculator/nmnaud1hrw to play with the parameters + // att is not used for now + float att=1.0/(1.0+0.8*dist*dist); + + vec3 surf2light = normalize(u_lightPos - a_vertex); + vec3 norm = normalize(normal); + float dcont=max(0.0,dot(norm,surf2light)); + + float ambient = 0.3; + float intensity = dcont + 0.3 + ambient; + + v_color = u_materialDiffuse * intensity; + + gl_Position = u_viewProjectionMatrix * u_modelMatrix * vec4(a_vertex, 1.0); +} +""" + +BASIC_VERTEX_SHADER_130 = """ +#version 130 + +uniform mat4 u_viewProjectionMatrix; +uniform mat4 u_modelMatrix; +uniform mat3 u_normalMatrix; +uniform vec3 u_lightPos; + +uniform vec4 u_materialDiffuse; + +in vec3 a_vertex; +in vec3 a_normal; + +out vec4 v_color; + +void main(void) +{ + // Now the normal is in world space, as we pass the light in world space. + vec3 normal = u_normalMatrix * a_normal; + + float dist = distance(a_vertex, u_lightPos); + + // go to https://www.desmos.com/calculator/nmnaud1hrw to play with the parameters + // att is not used for now + float att=1.0/(1.0+0.8*dist*dist); + + vec3 surf2light = normalize(u_lightPos - a_vertex); + vec3 norm = normalize(normal); + float dcont=max(0.0,dot(norm,surf2light)); + + float ambient = 0.3; + float intensity = dcont + 0.3 + ambient; + + v_color = u_materialDiffuse * intensity; + + gl_Position = u_viewProjectionMatrix * u_modelMatrix * vec4(a_vertex, 1.0); +} +""" + +BASIC_FRAGMENT_SHADER_120 = """ +#version 120 + +varying vec4 v_color; + +void main() { + gl_FragColor = v_color; +} +""" + +BASIC_FRAGMENT_SHADER_130 = """ +#version 130 + +in vec4 v_color; + +void main() { + gl_FragColor = v_color; +} +""" + +GOOCH_VERTEX_SHADER_120 = """ +#version 120 + +// attributes +attribute vec3 a_vertex; // xyz - position +attribute vec3 a_normal; // xyz - normal + +// uniforms +uniform mat4 u_modelMatrix; +uniform mat4 u_viewProjectionMatrix; +uniform mat3 u_normalMatrix; +uniform vec3 u_lightPos; +uniform vec3 u_camPos; + +// output data from vertex to fragment shader +varying vec3 o_normal; +varying vec3 o_lightVector; + +/////////////////////////////////////////////////////////////////// + +void main(void) +{ + // transform position and normal to world space + vec4 positionWorld = u_modelMatrix * vec4(a_vertex, 1.0); + vec3 normalWorld = u_normalMatrix * a_normal; + + // calculate and pass vectors required for lighting + o_lightVector = u_lightPos - positionWorld.xyz; + o_normal = normalWorld; + + // project world space position to the screen and output it + gl_Position = u_viewProjectionMatrix * positionWorld; +} +""" + +GOOCH_VERTEX_SHADER_130 = """ +#version 130 + +// attributes +in vec3 a_vertex; // xyz - position +in vec3 a_normal; // xyz - normal + +// uniforms +uniform mat4 u_modelMatrix; +uniform mat4 u_viewProjectionMatrix; +uniform mat3 u_normalMatrix; +uniform vec3 u_lightPos; +uniform vec3 u_camPos; + +// output data from vertex to fragment shader +out vec3 o_normal; +out vec3 o_lightVector; + +/////////////////////////////////////////////////////////////////// + +void main(void) +{ + // transform position and normal to world space + vec4 positionWorld = u_modelMatrix * vec4(a_vertex, 1.0); + vec3 normalWorld = u_normalMatrix * a_normal; + + // calculate and pass vectors required for lighting + o_lightVector = u_lightPos - positionWorld.xyz; + o_normal = normalWorld; + + // project world space position to the screen and output it + gl_Position = u_viewProjectionMatrix * positionWorld; +} +""" + +GOOCH_FRAGMENT_SHADER_120 = """ +#version 120 + +// data from vertex shader +varying vec3 o_normal; +varying vec3 o_lightVector; + +// diffuse color of the object +uniform vec4 u_materialDiffuse; +// cool color of gooch shading +uniform vec3 u_coolColor; +// warm color of gooch shading +uniform vec3 u_warmColor; +// how much to take from object color in final cool color +uniform float u_alpha; +// how much to take from object color in final warm color +uniform float u_beta; + +/////////////////////////////////////////////////////////// + +void main(void) +{ + // normlize vectors for lighting + vec3 normalVector = normalize(o_normal); + vec3 lightVector = normalize(o_lightVector); + // intensity of diffuse lighting [-1, 1] + float diffuseLighting = dot(lightVector, normalVector); + // map intensity of lighting from range [-1; 1] to [0, 1] + float interpolationValue = (1.0 + diffuseLighting)/2; + + ////////////////////////////////////////////////////////////////// + + // cool color mixed with color of the object + vec3 coolColorMod = u_coolColor + vec3(u_materialDiffuse) * u_alpha; + // warm color mixed with color of the object + vec3 warmColorMod = u_warmColor + vec3(u_materialDiffuse) * u_beta; + // interpolation of cool and warm colors according + // to lighting intensity. The lower the light intensity, + // the larger part of the cool color is used + vec3 colorOut = mix(coolColorMod, warmColorMod, interpolationValue); + + ////////////////////////////////////////////////////////////////// + + // save color + gl_FragColor.rgb = colorOut; + gl_FragColor.a = 1; +} +""" + +GOOCH_FRAGMENT_SHADER_130 = """ +#version 130 + +// data from vertex shader +in vec3 o_normal; +in vec3 o_lightVector; + +// diffuse color of the object +uniform vec4 u_materialDiffuse; +// cool color of gooch shading +uniform vec3 u_coolColor; +// warm color of gooch shading +uniform vec3 u_warmColor; +// how much to take from object color in final cool color +uniform float u_alpha; +// how much to take from object color in final warm color +uniform float u_beta; + +// output to framebuffer +out vec4 resultingColor; + +/////////////////////////////////////////////////////////// + +void main(void) +{ + // normlize vectors for lighting + vec3 normalVector = normalize(o_normal); + vec3 lightVector = normalize(o_lightVector); + // intensity of diffuse lighting [-1, 1] + float diffuseLighting = dot(lightVector, normalVector); + // map intensity of lighting from range [-1; 1] to [0, 1] + float interpolationValue = (1.0 + diffuseLighting)/2; + + ////////////////////////////////////////////////////////////////// + + // cool color mixed with color of the object + vec3 coolColorMod = u_coolColor + vec3(u_materialDiffuse) * u_alpha; + // warm color mixed with color of the object + vec3 warmColorMod = u_warmColor + vec3(u_materialDiffuse) * u_beta; + // interpolation of cool and warm colors according + // to lighting intensity. The lower the light intensity, + // the larger part of the cool color is used + vec3 colorOut = mix(coolColorMod, warmColorMod, interpolationValue); + + ////////////////////////////////////////////////////////////////// + + // save color + resultingColor.rgb = colorOut; + resultingColor.a = 1; +} +""" + +SILHOUETTE_VERTEX_SHADER_120 = """ +#version 120 + +attribute vec3 a_vertex; // xyz - position +attribute vec3 a_normal; // xyz - normal + +uniform mat4 u_modelMatrix; +uniform mat4 u_viewProjectionMatrix; +uniform mat4 u_modelViewMatrix; +uniform vec4 u_materialDiffuse; +uniform float u_bordersize; // width of the border + +varying vec4 v_color; + +void main(void){ + v_color = u_materialDiffuse; + float distToCamera = -(u_modelViewMatrix * vec4(a_vertex, 1.0)).z; + vec4 tPos = vec4(a_vertex + a_normal * u_bordersize * distToCamera, 1.0); + gl_Position = u_viewProjectionMatrix * u_modelMatrix * tPos; +} +""" + +SILHOUETTE_VERTEX_SHADER_130 = """ +#version 130 + +in vec3 a_vertex; // xyz - position +in vec3 a_normal; // xyz - normal + +uniform mat4 u_modelMatrix; +uniform mat4 u_viewProjectionMatrix; +uniform mat4 u_modelViewMatrix; +uniform vec4 u_materialDiffuse; +uniform float u_bordersize; // width of the border + +out vec4 v_color; + +void main(void){ + v_color = u_materialDiffuse; + float distToCamera = -(u_modelViewMatrix * vec4(a_vertex, 1.0)).z; + vec4 tPos = vec4(a_vertex + a_normal * u_bordersize * distToCamera, 1.0); + gl_Position = u_viewProjectionMatrix * u_modelMatrix * tPos; +} +""" +DEFAULT_CLIP_PLANE_NEAR = 0.001 +DEFAULT_CLIP_PLANE_FAR = 1000.0 + + +def get_world_transform(scene, node): + if node == scene.rootnode: + return numpy.identity(4, dtype=numpy.float32) + + parents = reversed(_get_parent_chain(scene, node, [])) + parent_transform = reduce(numpy.dot, [p.transformation for p in parents]) + return numpy.dot(parent_transform, node.transformation) + + +def _get_parent_chain(scene, node, parents): + parent = node.parent + + parents.append(parent) + + if parent == scene.rootnode: + return parents + + return _get_parent_chain(scene, parent, parents) + + +class DefaultCamera: + def __init__(self, w, h, fov): + self.name = "default camera" + self.type = CAMERA + self.clipplanenear = DEFAULT_CLIP_PLANE_NEAR + self.clipplanefar = DEFAULT_CLIP_PLANE_FAR + self.aspect = w / h + self.horizontalfov = fov * math.pi / 180 + self.transformation = numpy.array([[0.68, -0.32, 0.65, 7.48], + [0.73, 0.31, -0.61, -6.51], + [-0.01, 0.89, 0.44, 5.34], + [0., 0., 0., 1.]], dtype=numpy.float32) + + self.transformation = numpy.dot(self.transformation, ROTATION_180_X) + + def __str__(self): + return self.name + + +class PyAssimp3DViewer: + base_name = "PyASSIMP 3D viewer" + + def __init__(self, model, w=1024, h=768): + + self.w = w + self.h = h + + pygame.init() + pygame.display.set_caption(self.base_name) + pygame.display.set_mode((w, h), pygame.OPENGL | pygame.DOUBLEBUF) + + glClearColor(0.18, 0.18, 0.18, 1.0) + + shader_compilation_succeeded = False + try: + self.set_shaders_v130() + self.prepare_shaders() + except RuntimeError, message: + sys.stderr.write("%s\n" % message) + sys.stdout.write("Could not compile shaders in version 1.30, trying version 1.20\n") + + if not shader_compilation_succeeded: + self.set_shaders_v120() + self.prepare_shaders() + + self.scene = None + self.meshes = {} # stores the OpenGL vertex/faces/normals buffers pointers + + self.node2colorid = {} # stores a color ID for each node. Useful for mouse picking and visibility checking + self.colorid2node = {} # reverse dict of node2colorid + + self.currently_selected = None + self.moving = False + self.moving_situation = None + + self.default_camera = DefaultCamera(self.w, self.h, fov=70) + self.cameras = [self.default_camera] + + self.current_cam_index = 0 + self.current_cam = self.default_camera + self.set_camera_projection() + + self.load_model(model) + + # user interactions + self.focal_point = [0, 0, 0] + self.is_rotating = False + self.is_panning = False + self.is_zooming = False + + def set_shaders_v120(self): + self.BASIC_VERTEX_SHADER = BASIC_VERTEX_SHADER_120 + self.FLAT_VERTEX_SHADER = FLAT_VERTEX_SHADER_120 + self.SILHOUETTE_VERTEX_SHADER = SILHOUETTE_VERTEX_SHADER_120 + self.GOOCH_VERTEX_SHADER = GOOCH_VERTEX_SHADER_120 + + self.BASIC_FRAGMENT_SHADER = BASIC_FRAGMENT_SHADER_120 + self.GOOCH_FRAGMENT_SHADER = GOOCH_FRAGMENT_SHADER_120 + + def set_shaders_v130(self): + self.BASIC_VERTEX_SHADER = BASIC_VERTEX_SHADER_130 + self.FLAT_VERTEX_SHADER = FLAT_VERTEX_SHADER_130 + self.SILHOUETTE_VERTEX_SHADER = SILHOUETTE_VERTEX_SHADER_130 + self.GOOCH_VERTEX_SHADER = GOOCH_VERTEX_SHADER_130 + + self.BASIC_FRAGMENT_SHADER = BASIC_FRAGMENT_SHADER_130 + self.GOOCH_FRAGMENT_SHADER = GOOCH_FRAGMENT_SHADER_130 + + def prepare_shaders(self): + + ### Base shader + vertex = shaders.compileShader(self.BASIC_VERTEX_SHADER, GL_VERTEX_SHADER) + fragment = shaders.compileShader(self.BASIC_FRAGMENT_SHADER, GL_FRAGMENT_SHADER) + + self.shader = shaders.compileProgram(vertex, fragment) + + self.set_shader_accessors(('u_modelMatrix', + 'u_viewProjectionMatrix', + 'u_normalMatrix', + 'u_lightPos', + 'u_materialDiffuse'), + ('a_vertex', + 'a_normal'), self.shader) + + ### Flat shader + flatvertex = shaders.compileShader(self.FLAT_VERTEX_SHADER, GL_VERTEX_SHADER) + self.flatshader = shaders.compileProgram(flatvertex, fragment) + + self.set_shader_accessors(('u_modelMatrix', + 'u_viewProjectionMatrix', + 'u_materialDiffuse',), + ('a_vertex',), self.flatshader) + + ### Silhouette shader + silh_vertex = shaders.compileShader(self.SILHOUETTE_VERTEX_SHADER, GL_VERTEX_SHADER) + self.silhouette_shader = shaders.compileProgram(silh_vertex, fragment) + + self.set_shader_accessors(('u_modelMatrix', + 'u_viewProjectionMatrix', + 'u_modelViewMatrix', + 'u_materialDiffuse', + 'u_bordersize' # width of the silhouette + ), + ('a_vertex', + 'a_normal'), self.silhouette_shader) + + ### Gooch shader + gooch_vertex = shaders.compileShader(self.GOOCH_VERTEX_SHADER, GL_VERTEX_SHADER) + gooch_fragment = shaders.compileShader(self.GOOCH_FRAGMENT_SHADER, GL_FRAGMENT_SHADER) + self.gooch_shader = shaders.compileProgram(gooch_vertex, gooch_fragment) + + self.set_shader_accessors(('u_modelMatrix', + 'u_viewProjectionMatrix', + 'u_normalMatrix', + 'u_lightPos', + 'u_materialDiffuse', + 'u_coolColor', + 'u_warmColor', + 'u_alpha', + 'u_beta' + ), + ('a_vertex', + 'a_normal'), self.gooch_shader) + + @staticmethod + def set_shader_accessors(uniforms, attributes, shader): + # add accessors to the shaders uniforms and attributes + for uniform in uniforms: + location = glGetUniformLocation(shader, uniform) + if location in (None, -1): + raise RuntimeError('No uniform: %s (maybe it is not used ' + 'anymore and has been optimized out by' + ' the shader compiler)' % uniform) + setattr(shader, uniform, location) + + for attribute in attributes: + location = glGetAttribLocation(shader, attribute) + if location in (None, -1): + raise RuntimeError('No attribute: %s' % attribute) + setattr(shader, attribute, location) + + @staticmethod + def prepare_gl_buffers(mesh): + + mesh.gl = {} + + # Fill the buffer for vertex and normals positions + v = numpy.array(mesh.vertices, 'f') + n = numpy.array(mesh.normals, 'f') + + mesh.gl["vbo"] = vbo.VBO(numpy.hstack((v, n))) + + # Fill the buffer for vertex positions + mesh.gl["faces"] = glGenBuffers(1) + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mesh.gl["faces"]) + glBufferData(GL_ELEMENT_ARRAY_BUFFER, + numpy.array(mesh.faces, dtype=numpy.int32), + GL_STATIC_DRAW) + + mesh.gl["nbfaces"] = len(mesh.faces) + + # Unbind buffers + glBindBuffer(GL_ARRAY_BUFFER, 0) + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0) + + @staticmethod + def get_rgb_from_colorid(colorid): + r = (colorid >> 0) & 0xff + g = (colorid >> 8) & 0xff + b = (colorid >> 16) & 0xff + + return r, g, b + + def get_color_id(self): + id = random.randint(0, 256 * 256 * 256) + if id not in self.colorid2node: + return id + else: + return self.get_color_id() + + def glize(self, scene, node): + + logger.info("Loading node <%s>" % node) + node.selected = True if self.currently_selected and self.currently_selected == node else False + + node.transformation = node.transformation.astype(numpy.float32) + + if node.meshes: + node.type = MESH + colorid = self.get_color_id() + self.colorid2node[colorid] = node + self.node2colorid[node.name] = colorid + + elif node.name in [c.name for c in scene.cameras]: + + # retrieve the ASSIMP camera object + [cam] = [c for c in scene.cameras if c.name == node.name] + node.type = CAMERA + logger.info("Added camera <%s>" % node.name) + logger.info("Camera position: %.3f, %.3f, %.3f" % tuple(node.transformation[:, 3][:3].tolist())) + self.cameras.append(node) + node.clipplanenear = cam.clipplanenear + node.clipplanefar = cam.clipplanefar + + if numpy.allclose(cam.lookat, [0, 0, -1]) and numpy.allclose(cam.up, [0, 1, 0]): # Cameras in .blend files + + # Rotate by 180deg around X to have Z pointing forward + node.transformation = numpy.dot(node.transformation, ROTATION_180_X) + else: + raise RuntimeError( + "I do not know how to normalize this camera orientation: lookat=%s, up=%s" % (cam.lookat, cam.up)) + + if cam.aspect == 0.0: + logger.warning("Camera aspect not set. Setting to default 4:3") + node.aspect = 1.333 + else: + node.aspect = cam.aspect + + node.horizontalfov = cam.horizontalfov + + else: + node.type = ENTITY + + for child in node.children: + self.glize(scene, child) + + def load_model(self, path, postprocess=aiProcessPreset_TargetRealtime_MaxQuality): + logger.info("Loading model:" + path + "...") + + if postprocess: + self.scene = pyassimp.load(path, processing=postprocess) + else: + self.scene = pyassimp.load(path) + logger.info("Done.") + + scene = self.scene + # log some statistics + logger.info(" meshes: %d" % len(scene.meshes)) + logger.info(" total faces: %d" % sum([len(mesh.faces) for mesh in scene.meshes])) + logger.info(" materials: %d" % len(scene.materials)) + self.bb_min, self.bb_max = get_bounding_box(self.scene) + logger.info(" bounding box:" + str(self.bb_min) + " - " + str(self.bb_max)) + + self.scene_center = [(a + b) / 2. for a, b in zip(self.bb_min, self.bb_max)] + + for index, mesh in enumerate(scene.meshes): + self.prepare_gl_buffers(mesh) + + self.glize(scene, scene.rootnode) + + # Finally release the model + pyassimp.release(scene) + logger.info("Ready for 3D rendering!") + + def cycle_cameras(self): + + self.current_cam_index = (self.current_cam_index + 1) % len(self.cameras) + self.current_cam = self.cameras[self.current_cam_index] + self.set_camera_projection(self.current_cam) + logger.info("Switched to camera <%s>" % self.current_cam) + + def set_overlay_projection(self): + glViewport(0, 0, self.w, self.h) + glMatrixMode(GL_PROJECTION) + glLoadIdentity() + glOrtho(0.0, self.w - 1.0, 0.0, self.h - 1.0, -1.0, 1.0) + glMatrixMode(GL_MODELVIEW) + glLoadIdentity() + + def set_camera_projection(self, camera=None): + + if not camera: + camera = self.current_cam + + znear = camera.clipplanenear or DEFAULT_CLIP_PLANE_NEAR + zfar = camera.clipplanefar or DEFAULT_CLIP_PLANE_FAR + aspect = camera.aspect + fov = camera.horizontalfov + + glMatrixMode(GL_PROJECTION) + glLoadIdentity() + + # Compute gl frustrum + tangent = math.tan(fov / 2.) + h = znear * tangent + w = h * aspect + + # params: left, right, bottom, top, near, far + glFrustum(-w, w, -h, h, znear, zfar) + # equivalent to: + # gluPerspective(fov * 180/math.pi, aspect, znear, zfar) + + self.projection_matrix = glGetFloatv(GL_PROJECTION_MATRIX).transpose() + + glMatrixMode(GL_MODELVIEW) + glLoadIdentity() + + def render_colors(self): + + glEnable(GL_DEPTH_TEST) + glDepthFunc(GL_LEQUAL) + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) + glEnable(GL_CULL_FACE) + + glUseProgram(self.flatshader) + + glUniformMatrix4fv(self.flatshader.u_viewProjectionMatrix, 1, GL_TRUE, + numpy.dot(self.projection_matrix, self.view_matrix)) + + self.recursive_render(self.scene.rootnode, self.flatshader, mode=COLORS) + + glUseProgram(0) + + def get_hovered_node(self, mousex, mousey): + """ + Attention: The performances of this method relies heavily on the size of the display! + """ + + # mouse out of the window? + if mousex < 0 or mousex >= self.w or mousey < 0 or mousey >= self.h: + return None + + self.render_colors() + # Capture image from the OpenGL buffer + buf = (GLubyte * (3 * self.w * self.h))(0) + glReadPixels(0, 0, self.w, self.h, GL_RGB, GL_UNSIGNED_BYTE, buf) + + # Reinterpret the RGB pixel buffer as a 1-D array of 24bits colors + a = numpy.ndarray(len(buf), numpy.dtype('>u1'), buf) + colors = numpy.zeros(len(buf) // 3, numpy.dtype('u1')[i::3] + + colorid = colors[mousex + mousey * self.w] + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + + if colorid in self.colorid2node: + return self.colorid2node[colorid] + + def render(self, wireframe=False, twosided=False): + + glEnable(GL_DEPTH_TEST) + glDepthFunc(GL_LEQUAL) + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE if wireframe else GL_FILL) + glDisable(GL_CULL_FACE) if twosided else glEnable(GL_CULL_FACE) + + self.render_grid() + + self.recursive_render(self.scene.rootnode, None, mode=HELPERS) + + ### First, the silhouette + + if False: + shader = self.silhouette_shader + + # glDepthMask(GL_FALSE) + glCullFace(GL_FRONT) # cull front faces + + glUseProgram(shader) + glUniform1f(shader.u_bordersize, 0.01) + + glUniformMatrix4fv(shader.u_viewProjectionMatrix, 1, GL_TRUE, + numpy.dot(self.projection_matrix, self.view_matrix)) + + self.recursive_render(self.scene.rootnode, shader, mode=SILHOUETTE) + + glUseProgram(0) + + ### Then, inner shading + # glDepthMask(GL_TRUE) + glCullFace(GL_BACK) + + use_gooch = False + if use_gooch: + shader = self.gooch_shader + + glUseProgram(shader) + glUniform3f(shader.u_lightPos, -.5, -.5, .5) + + ##### GOOCH specific + glUniform3f(shader.u_coolColor, 159.0 / 255, 148.0 / 255, 255.0 / 255) + glUniform3f(shader.u_warmColor, 255.0 / 255, 75.0 / 255, 75.0 / 255) + glUniform1f(shader.u_alpha, .25) + glUniform1f(shader.u_beta, .25) + ######### + else: + shader = self.shader + glUseProgram(shader) + glUniform3f(shader.u_lightPos, -.5, -.5, .5) + + glUniformMatrix4fv(shader.u_viewProjectionMatrix, 1, GL_TRUE, + numpy.dot(self.projection_matrix, self.view_matrix)) + + self.recursive_render(self.scene.rootnode, shader) + + glUseProgram(0) + + def render_axis(self, + transformation=numpy.identity(4, dtype=numpy.float32), + label=None, + size=0.2, + selected=False): + m = transformation.transpose() # OpenGL row major + + glPushMatrix() + glMultMatrixf(m) + + glLineWidth(3 if selected else 1) + + size = 2 * size if selected else size + + glBegin(GL_LINES) + + # draw line for x axis + glColor3f(1.0, 0.0, 0.0) + glVertex3f(0.0, 0.0, 0.0) + glVertex3f(size, 0.0, 0.0) + + # draw line for y axis + glColor3f(0.0, 1.0, 0.0) + glVertex3f(0.0, 0.0, 0.0) + glVertex3f(0.0, size, 0.0) + + # draw line for Z axis + glColor3f(0.0, 0.0, 1.0) + glVertex3f(0.0, 0.0, 0.0) + glVertex3f(0.0, 0.0, size) + + glEnd() + + if label: + self.showtext(label) + + glPopMatrix() + + @staticmethod + def render_camera(camera, transformation): + + m = transformation.transpose() # OpenGL row major + + aspect = camera.aspect + + u = 0.1 # unit size (in m) + l = 3 * u # lenght of the camera cone + f = 3 * u # aperture of the camera cone + + glPushMatrix() + glMultMatrixf(m) + + glLineWidth(2) + glBegin(GL_LINE_STRIP) + + glColor3f(.2, .2, .2) + + glVertex3f(u, u, -u) + glVertex3f(u, -u, -u) + glVertex3f(-u, -u, -u) + glVertex3f(-u, u, -u) + glVertex3f(u, u, -u) + + glVertex3f(u, u, 0.0) + glVertex3f(u, -u, 0.0) + glVertex3f(-u, -u, 0.0) + glVertex3f(-u, u, 0.0) + glVertex3f(u, u, 0.0) + + glVertex3f(f * aspect, f, l) + glVertex3f(f * aspect, -f, l) + glVertex3f(-f * aspect, -f, l) + glVertex3f(-f * aspect, f, l) + glVertex3f(f * aspect, f, l) + + glEnd() + + glBegin(GL_LINE_STRIP) + glVertex3f(u, -u, -u) + glVertex3f(u, -u, 0.0) + glVertex3f(f * aspect, -f, l) + glEnd() + + glBegin(GL_LINE_STRIP) + glVertex3f(-u, -u, -u) + glVertex3f(-u, -u, 0.0) + glVertex3f(-f * aspect, -f, l) + glEnd() + + glBegin(GL_LINE_STRIP) + glVertex3f(-u, u, -u) + glVertex3f(-u, u, 0.0) + glVertex3f(-f * aspect, f, l) + glEnd() + + glPopMatrix() + + @staticmethod + def render_grid(): + + glLineWidth(1) + glColor3f(0.5, 0.5, 0.5) + glBegin(GL_LINES) + for i in range(-10, 11): + glVertex3f(i, -10.0, 0.0) + glVertex3f(i, 10.0, 0.0) + + for i in range(-10, 11): + glVertex3f(-10.0, i, 0.0) + glVertex3f(10.0, i, 0.0) + glEnd() + + def recursive_render(self, node, shader, mode=BASE, with_normals=True): + """ Main recursive rendering method. + """ + + normals = with_normals + + if mode == COLORS: + normals = False + + + if not hasattr(node, "selected"): + node.selected = False + + m = get_world_transform(self.scene, node) + + # HELPERS mode + ### + if mode == HELPERS: + # if node.type == ENTITY: + self.render_axis(m, + label=node.name if node != self.scene.rootnode else None, + selected=node.selected if hasattr(node, "selected") else False) + + if node.type == CAMERA: + self.render_camera(node, m) + + for child in node.children: + self.recursive_render(child, shader, mode) + + return + + # Mesh rendering modes + ### + if node.type == MESH: + + for mesh in node.meshes: + + stride = 24 # 6 * 4 bytes + + if node.selected and mode == SILHOUETTE: + glUniform4f(shader.u_materialDiffuse, 1.0, 0.0, 0.0, 1.0) + glUniformMatrix4fv(shader.u_modelViewMatrix, 1, GL_TRUE, + numpy.dot(self.view_matrix, m)) + + else: + if mode == COLORS: + colorid = self.node2colorid[node.name] + r, g, b = self.get_rgb_from_colorid(colorid) + glUniform4f(shader.u_materialDiffuse, r / 255.0, g / 255.0, b / 255.0, 1.0) + elif mode == SILHOUETTE: + glUniform4f(shader.u_materialDiffuse, .0, .0, .0, 1.0) + else: + if node.selected: + diffuse = (1.0, 0.0, 0.0, 1.0) # selected nodes in red + else: + diffuse = mesh.material.properties["diffuse"] + if len(diffuse) == 3: # RGB instead of expected RGBA + diffuse.append(1.0) + glUniform4f(shader.u_materialDiffuse, *diffuse) + # if ambient: + # glUniform4f( shader.Material_ambient, *mat["ambient"] ) + + if mode == BASE: # not in COLORS or SILHOUETTE + normal_matrix = linalg.inv(numpy.dot(self.view_matrix, m)[0:3, 0:3]).transpose() + glUniformMatrix3fv(shader.u_normalMatrix, 1, GL_TRUE, normal_matrix) + + glUniformMatrix4fv(shader.u_modelMatrix, 1, GL_TRUE, m) + + vbo = mesh.gl["vbo"] + vbo.bind() + + glEnableVertexAttribArray(shader.a_vertex) + if normals: + glEnableVertexAttribArray(shader.a_normal) + + glVertexAttribPointer( + shader.a_vertex, + 3, GL_FLOAT, False, stride, vbo + ) + + if normals: + glVertexAttribPointer( + shader.a_normal, + 3, GL_FLOAT, False, stride, vbo + 12 + ) + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mesh.gl["faces"]) + glDrawElements(GL_TRIANGLES, mesh.gl["nbfaces"] * 3, GL_UNSIGNED_INT, None) + + vbo.unbind() + glDisableVertexAttribArray(shader.a_vertex) + + if normals: + glDisableVertexAttribArray(shader.a_normal) + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0) + + for child in node.children: + self.recursive_render(child, shader, mode) + + + def switch_to_overlay(self): + glPushMatrix() + self.set_overlay_projection() + + def switch_from_overlay(self): + self.set_camera_projection() + glPopMatrix() + + def select_node(self, node): + self.currently_selected = node + self.update_node_select(self.scene.rootnode) + + def update_node_select(self, node): + if node is self.currently_selected: + node.selected = True + else: + node.selected = False + + for child in node.children: + self.update_node_select(child) + + def loop(self): + + pygame.display.flip() + + if not self.process_events(): + return False # ESC has been pressed + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + + return True + + def process_events(self): + + LEFT_BUTTON = 1 + MIDDLE_BUTTON = 2 + RIGHT_BUTTON = 3 + WHEEL_UP = 4 + WHEEL_DOWN = 5 + + dx, dy = pygame.mouse.get_rel() + mousex, mousey = pygame.mouse.get_pos() + + zooming_one_shot = False + + ok = True + + for evt in pygame.event.get(): + if evt.type == pygame.MOUSEBUTTONDOWN and evt.button == LEFT_BUTTON: + hovered = self.get_hovered_node(mousex, self.h - mousey) + if hovered: + if self.currently_selected and self.currently_selected == hovered: + self.select_node(None) + else: + logger.info("Node %s selected" % hovered) + self.select_node(hovered) + else: + self.is_rotating = True + if evt.type == pygame.MOUSEBUTTONUP and evt.button == LEFT_BUTTON: + self.is_rotating = False + + if evt.type == pygame.MOUSEBUTTONDOWN and evt.button == MIDDLE_BUTTON: + self.is_panning = True + if evt.type == pygame.MOUSEBUTTONUP and evt.button == MIDDLE_BUTTON: + self.is_panning = False + + if evt.type == pygame.MOUSEBUTTONDOWN and evt.button == RIGHT_BUTTON: + self.is_zooming = True + if evt.type == pygame.MOUSEBUTTONUP and evt.button == RIGHT_BUTTON: + self.is_zooming = False + + if evt.type == pygame.MOUSEBUTTONDOWN and evt.button in [WHEEL_UP, WHEEL_DOWN]: + zooming_one_shot = True + self.is_zooming = True + dy = -10 if evt.button == WHEEL_UP else 10 + + if evt.type == pygame.KEYDOWN: + ok = (ok and self.process_keystroke(evt.key, evt.mod)) + + self.controls_3d(dx, dy, zooming_one_shot) + + return ok + + def process_keystroke(self, key, mod): + + # process arrow keys if an object is selected + if self.currently_selected: + up = 0 + strafe = 0 + + if key == pygame.K_UP: + up = 1 + if key == pygame.K_DOWN: + up = -1 + if key == pygame.K_LEFT: + strafe = -1 + if key == pygame.K_RIGHT: + strafe = 1 + + self.move_selected_node(up, strafe) + + if key == pygame.K_f: + pygame.display.toggle_fullscreen() + + if key == pygame.K_TAB: + self.cycle_cameras() + + if key in [pygame.K_ESCAPE, pygame.K_q]: + return False + + return True + + def controls_3d(self, dx, dy, zooming_one_shot=False): + + CAMERA_TRANSLATION_FACTOR = 0.01 + CAMERA_ROTATION_FACTOR = 0.01 + + if not (self.is_rotating or self.is_panning or self.is_zooming): + return + + current_pos = self.current_cam.transformation[:3, 3].copy() + distance = numpy.linalg.norm(self.focal_point - current_pos) + + if self.is_rotating: + """ Orbiting the camera is implemented the following way: + + - the rotation is split into a rotation around the *world* Z axis + (controlled by the horizontal mouse motion along X) and a + rotation around the *X* axis of the camera (pitch) *shifted to + the focal origin* (the world origin for now). This is controlled + by the vertical motion of the mouse (Y axis). + + - as a result, the resulting transformation of the camera in the + world frame C' is: + C' = (T · Rx · T⁻¹ · (Rz · C)⁻¹)⁻¹ + + where: + - C is the original camera transformation in the world frame, + - Rz is the rotation along the Z axis (in the world frame) + - T is the translation camera -> world (ie, the inverse of the + translation part of C + - Rx is the rotation around X in the (translated) camera frame + """ + + rotation_camera_x = dy * CAMERA_ROTATION_FACTOR + rotation_world_z = dx * CAMERA_ROTATION_FACTOR + world_z_rotation = transformations.euler_matrix(0, 0, rotation_world_z) + cam_x_rotation = transformations.euler_matrix(rotation_camera_x, 0, 0) + + after_world_z_rotation = numpy.dot(world_z_rotation, self.current_cam.transformation) + + inverse_transformation = transformations.inverse_matrix(after_world_z_rotation) + + translation = transformations.translation_matrix( + transformations.decompose_matrix(inverse_transformation)[3]) + inverse_translation = transformations.inverse_matrix(translation) + + new_inverse = numpy.dot(inverse_translation, inverse_transformation) + new_inverse = numpy.dot(cam_x_rotation, new_inverse) + new_inverse = numpy.dot(translation, new_inverse) + + self.current_cam.transformation = transformations.inverse_matrix(new_inverse).astype(numpy.float32) + + if self.is_panning: + tx = -dx * CAMERA_TRANSLATION_FACTOR * distance + ty = dy * CAMERA_TRANSLATION_FACTOR * distance + cam_transform = transformations.translation_matrix((tx, ty, 0)).astype(numpy.float32) + self.current_cam.transformation = numpy.dot(self.current_cam.transformation, cam_transform) + + if self.is_zooming: + tz = dy * CAMERA_TRANSLATION_FACTOR * distance + cam_transform = transformations.translation_matrix((0, 0, tz)).astype(numpy.float32) + self.current_cam.transformation = numpy.dot(self.current_cam.transformation, cam_transform) + + if zooming_one_shot: + self.is_zooming = False + + self.update_view_camera() + + def update_view_camera(self): + + self.view_matrix = linalg.inv(self.current_cam.transformation) + + # Rotate by 180deg around X to have Z pointing backward (OpenGL convention) + self.view_matrix = numpy.dot(ROTATION_180_X, self.view_matrix) + + glMatrixMode(GL_MODELVIEW) + glLoadIdentity() + glMultMatrixf(self.view_matrix.transpose()) + + def move_selected_node(self, up, strafe): + self.currently_selected.transformation[0][3] += strafe + self.currently_selected.transformation[2][3] += up + + @staticmethod + def showtext(text, x=0, y=0, z=0, size=20): + + # TODO: alpha blending does not work... + # glEnable(GL_BLEND) + # glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) + + font = pygame.font.Font(None, size) + text_surface = font.render(text, True, (10, 10, 10, 255), + (255 * 0.18, 255 * 0.18, 255 * 0.18, 0)) + text_data = pygame.image.tostring(text_surface, "RGBA", True) + glRasterPos3d(x, y, z) + glDrawPixels(text_surface.get_width(), + text_surface.get_height(), + GL_RGBA, GL_UNSIGNED_BYTE, + text_data) + + # glDisable(GL_BLEND) + + +def main(model, width, height): + app = PyAssimp3DViewer(model, w=width, h=height) + + clock = pygame.time.Clock() + + while app.loop(): + + app.update_view_camera() + + ## Main rendering + app.render() + + ## GUI text display + app.switch_to_overlay() + app.showtext("Active camera: %s" % str(app.current_cam), 10, app.h - 30) + if app.currently_selected: + app.showtext("Selected node: %s" % app.currently_selected, 10, app.h - 50) + pos = app.h - 70 + + app.showtext("(%sm, %sm, %sm)" % (app.currently_selected.transformation[0, 3], + app.currently_selected.transformation[1, 3], + app.currently_selected.transformation[2, 3]), 30, pos) + + app.switch_from_overlay() + + # Make sure we do not go over 30fps + clock.tick(30) + + logger.info("Quitting! Bye bye!") + + +######################################################################### +######################################################################### + +if __name__ == '__main__': + if not len(sys.argv) > 1: + print("Usage: " + __file__ + " ") + sys.exit(2) + + main(model=sys.argv[1], width=1024, height=768) From 7182f89a518482c4d943ea532c2249883b8037ef Mon Sep 17 00:00:00 2001 From: Tammo Hinrichs Date: Thu, 24 Aug 2017 14:40:53 +0200 Subject: [PATCH 060/305] FBX importer: Use actual min/max of animation keyframes when start/stop time is missing --- code/FBXConverter.cpp | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index ebe7ac4e6..ae98d24fd 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -2368,8 +2368,13 @@ void Converter::ConvertAnimationStack( const AnimationStack& st ) int64_t start_time = st.LocalStart(); int64_t stop_time = st.LocalStop(); - double start_timeF = CONVERT_FBX_TIME( start_time ); - double stop_timeF = CONVERT_FBX_TIME( stop_time ); + bool has_local_startstop = start_time != 0 || stop_time != 0; + if ( !has_local_startstop ) { + // no time range given, so accept every keyframe and use the actual min/max time + // the +/- 20000 is a safety because GenerateNodeAnimations uses an epsilon of 10000 + start_time = INT64_MIN + 20000; + stop_time = INT64_MAX - 20000; + } try { for( const NodeMap::value_type& kv : node_map ) { @@ -2401,27 +2406,23 @@ void Converter::ConvertAnimationStack( const AnimationStack& st ) return; } - //adjust relative timing for animation - { - double start_fps = start_timeF * anim_fps; + double start_time_fps = has_local_startstop ? (CONVERT_FBX_TIME(start_time) * anim_fps) : min_time; + double stop_time_fps = has_local_startstop ? (CONVERT_FBX_TIME(stop_time) * anim_fps) : max_time; - for ( unsigned int c = 0; c < anim->mNumChannels; c++ ) - { - aiNodeAnim* channel = anim->mChannels[ c ]; - for ( uint32_t i = 0; i < channel->mNumPositionKeys; i++ ) - channel->mPositionKeys[ i ].mTime -= start_fps; - for ( uint32_t i = 0; i < channel->mNumRotationKeys; i++ ) - channel->mRotationKeys[ i ].mTime -= start_fps; - for ( uint32_t i = 0; i < channel->mNumScalingKeys; i++ ) - channel->mScalingKeys[ i ].mTime -= start_fps; - } - - max_time -= min_time; + // adjust relative timing for animation + for ( unsigned int c = 0; c < anim->mNumChannels; c++ ) { + aiNodeAnim* channel = anim->mChannels[ c ]; + for ( uint32_t i = 0; i < channel->mNumPositionKeys; i++ ) + channel->mPositionKeys[ i ].mTime -= start_time_fps; + for ( uint32_t i = 0; i < channel->mNumRotationKeys; i++ ) + channel->mRotationKeys[ i ].mTime -= start_time_fps; + for ( uint32_t i = 0; i < channel->mNumScalingKeys; i++ ) + channel->mScalingKeys[ i ].mTime -= start_time_fps; } // for some mysterious reason, mDuration is simply the maximum key -- the // validator always assumes animations to start at zero. - anim->mDuration = ( stop_timeF - start_timeF ) * anim_fps; + anim->mDuration = stop_time_fps - start_time_fps; anim->mTicksPerSecond = anim_fps; } From e90c615c2714d07acaae7c7e66fbe15873e95b6f Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 24 Aug 2017 17:03:41 +0200 Subject: [PATCH 061/305] Update Readme.md use new patreon badge. --- Readme.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 5a59d0364..88dc6fbc3 100644 --- a/Readme.md +++ b/Readme.md @@ -8,7 +8,6 @@ A library to import and export various 3d-model-formats including scene-post-pro Coverity Scan Build Status -Patreon donate button [![Coverage Status](https://coveralls.io/repos/github/assimp/assimp/badge.svg?branch=master)](https://coveralls.io/github/assimp/assimp?branch=master) [![Join the chat at https://gitter.im/assimp/assimp](https://badges.gitter.im/assimp/assimp.svg)](https://gitter.im/assimp/assimp?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -19,6 +18,11 @@ Additionally, assimp features various __mesh post processing tools__: normals an This is the development repo containing the latest features and bugfixes. For productive use though, we recommend one of the stable releases available from [Github Assimp Releases](https://github.com/assimp/assimp/releases). +Monthly donations via Patreon: +
[![Patreon](https://cloud.githubusercontent.com/assets/8225057/5990484/70413560-a9ab-11e4-8942-1a63607c0b00.png)](http://www.patreon.com/assimp) + +
+ One-off donations via PayPal:
[![PayPal](https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=4JRJVPXC4QJM4) From 80489963a15de3eea59810dbdbdeabdd6c103501 Mon Sep 17 00:00:00 2001 From: Tammo Hinrichs Date: Thu, 24 Aug 2017 17:18:54 +0200 Subject: [PATCH 062/305] FBX importer: don't rely ont INT64_MIN / ..MAX macros --- code/FBXConverter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index ae98d24fd..bbf0e4e11 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -2371,9 +2371,9 @@ void Converter::ConvertAnimationStack( const AnimationStack& st ) bool has_local_startstop = start_time != 0 || stop_time != 0; if ( !has_local_startstop ) { // no time range given, so accept every keyframe and use the actual min/max time - // the +/- 20000 is a safety because GenerateNodeAnimations uses an epsilon of 10000 - start_time = INT64_MIN + 20000; - stop_time = INT64_MAX - 20000; + // the numbers are INT64_MIN/MAX, the 20000 is for safety because GenerateNodeAnimations uses an epsilon of 10000 + start_time = -9223372036854775807i64 + 20000; + stop_time = 9223372036854775807i64 - 20000; } try { From 9a12b6ef0bbff90a69030a2cfa7a8b8756b904b5 Mon Sep 17 00:00:00 2001 From: Tammo Hinrichs Date: Fri, 25 Aug 2017 11:17:07 +0200 Subject: [PATCH 063/305] FBX importer: Back to INT64_MIN but include also. --- code/FBXConverter.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index bbf0e4e11..d6b21daed 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -61,6 +61,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include + namespace Assimp { namespace FBX { @@ -2371,9 +2373,9 @@ void Converter::ConvertAnimationStack( const AnimationStack& st ) bool has_local_startstop = start_time != 0 || stop_time != 0; if ( !has_local_startstop ) { // no time range given, so accept every keyframe and use the actual min/max time - // the numbers are INT64_MIN/MAX, the 20000 is for safety because GenerateNodeAnimations uses an epsilon of 10000 - start_time = -9223372036854775807i64 + 20000; - stop_time = 9223372036854775807i64 - 20000; + // the 20000 is for safety because GenerateNodeAnimations uses an epsilon of 10000 + start_time = INT64_MIN + 20000; + stop_time = INT64_MAX - 20000; } try { From 12a28d33ce205c9ed61fa14e5687e679f820d6e4 Mon Sep 17 00:00:00 2001 From: Tammo Hinrichs Date: Fri, 25 Aug 2017 12:14:03 +0200 Subject: [PATCH 064/305] FBX importer: try a constant again (ll suffix this time) --- code/FBXConverter.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index d6b21daed..f53ae0218 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -61,8 +61,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#include - namespace Assimp { namespace FBX { @@ -2373,9 +2371,9 @@ void Converter::ConvertAnimationStack( const AnimationStack& st ) bool has_local_startstop = start_time != 0 || stop_time != 0; if ( !has_local_startstop ) { // no time range given, so accept every keyframe and use the actual min/max time - // the 20000 is for safety because GenerateNodeAnimations uses an epsilon of 10000 - start_time = INT64_MIN + 20000; - stop_time = INT64_MAX - 20000; + // the numbers are INT64_MIN/MAX, the 20000 is for safety because GenerateNodeAnimations uses an epsilon of 10000 + start_time = -9223372036854775807ll + 20000; + stop_time = 9223372036854775807ll - 20000; } try { From f8ef94095a460beacfd7574d74237dde4ae1bc9c Mon Sep 17 00:00:00 2001 From: Alexander Gessler Date: Tue, 29 Aug 2017 15:41:25 +0200 Subject: [PATCH 065/305] Update version check in FBX reader to check for version >= 7500 instead of exactly 7500. --- code/FBXBinaryTokenizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/FBXBinaryTokenizer.cpp b/code/FBXBinaryTokenizer.cpp index d388c6446..46e85d888 100644 --- a/code/FBXBinaryTokenizer.cpp +++ b/code/FBXBinaryTokenizer.cpp @@ -445,7 +445,7 @@ void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int le const uint8_t unknown_4 = ReadByte(input, cursor, input + length); const uint8_t unknown_5 = ReadByte(input, cursor, input + length); const uint32_t version = ReadWord(input, cursor, input + length); - const bool is64bits = version == 7500; + const bool is64bits = version >= 7500; while (cursor < input + length) { if (!ReadScope(output_tokens, input, cursor, input + length, is64bits)) { From 1e3a9ded112cb8fb8a438a1d57111a6683f22c13 Mon Sep 17 00:00:00 2001 From: THISISAGOODNAME Date: Wed, 30 Aug 2017 23:18:36 +0800 Subject: [PATCH 066/305] Fix assimp_qt_viewer build failed on OSX --- tools/assimp_qt_viewer/glview.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/assimp_qt_viewer/glview.cpp b/tools/assimp_qt_viewer/glview.cpp index caf4ef320..429f2ca99 100644 --- a/tools/assimp_qt_viewer/glview.cpp +++ b/tools/assimp_qt_viewer/glview.cpp @@ -6,7 +6,11 @@ #include "glview.hpp" // Header files, OpenGL. -#include +#if defined(__APPLE__) +# include +#else +# include +#endif // Header files, DevIL. #include From 00e3b03c5ba2fbb0af21d42a9b6a54b512b19ba9 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 31 Aug 2017 11:04:13 +0200 Subject: [PATCH 067/305] closes https://github.com/assimp/assimp/issues/1406: fix merge issue + improve 3MF-tests --- code/D3MFImporter.cpp | 8 ++------ test/unit/utD3MFImportExport.cpp | 11 +++++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/code/D3MFImporter.cpp b/code/D3MFImporter.cpp index 0f06d4a1e..e97e22859 100644 --- a/code/D3MFImporter.cpp +++ b/code/D3MFImporter.cpp @@ -95,14 +95,10 @@ public: XmlSerializer(XmlReader* xmlReader) : xmlReader(xmlReader) { - + // empty } - void ImportXml(aiScene* scene) - { - - scene->mFlags |= AI_SCENE_FLAGS_NON_VERBOSE_FORMAT; - + void ImportXml(aiScene* scene) { scene->mRootNode = new aiNode(); std::vector children; diff --git a/test/unit/utD3MFImportExport.cpp b/test/unit/utD3MFImportExport.cpp index 7b7e8fb11..b77f44f86 100644 --- a/test/unit/utD3MFImportExport.cpp +++ b/test/unit/utD3MFImportExport.cpp @@ -43,13 +43,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "AbstractImportExportBase.h" #include +#include class utD3MFImporterExporter : public AbstractImportExportBase { public: - virtual bool importerTest() { + bool importerTest() override { Assimp::Importer importer; const aiScene *scene = importer.ReadFile(ASSIMP_TEST_MODELS_DIR "/3MF/box.3mf", 0); - return nullptr != scene; + EXPECT_EQ( 1u, scene->mNumMeshes ); + aiMesh *mesh = scene->mMeshes[ 0 ]; + EXPECT_NE( nullptr, mesh ); + EXPECT_EQ( 12u, mesh->mNumFaces ); + EXPECT_EQ( 8u, mesh->mNumVertices ); + + return ( nullptr != scene ); } }; From b4da9c4f56b10aff0a27da5561931aec623ac8b2 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 31 Aug 2017 11:30:31 +0200 Subject: [PATCH 068/305] Fix review findings: remove unused includes --- code/Q3BSPZipArchive.cpp | 5 ----- code/Q3BSPZipArchive.h | 1 - 2 files changed, 6 deletions(-) diff --git a/code/Q3BSPZipArchive.cpp b/code/Q3BSPZipArchive.cpp index 86f399659..7428c04c3 100644 --- a/code/Q3BSPZipArchive.cpp +++ b/code/Q3BSPZipArchive.cpp @@ -39,14 +39,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ - - #ifndef ASSIMP_BUILD_NO_Q3BSP_IMPORTER #include "Q3BSPZipArchive.h" -#include #include -#include #include namespace Assimp { @@ -137,7 +133,6 @@ zlib_filefunc_def IOSystem2Unzip::get(IOSystem* pIOHandler) { return mapping; } -// ------------------------------------------------------------------------------------------------ ZipFile::ZipFile(size_t size) : m_Size(size) { ai_assert(m_Size != 0); diff --git a/code/Q3BSPZipArchive.h b/code/Q3BSPZipArchive.h index 280c44fc0..5430bac6f 100644 --- a/code/Q3BSPZipArchive.h +++ b/code/Q3BSPZipArchive.h @@ -44,7 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include #include #include #include From 11f70e21404ac9d199088c36bbdbc6c2871ae4a8 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 31 Aug 2017 11:39:10 +0200 Subject: [PATCH 069/305] Tests: add 3D-importer test. --- test/CMakeLists.txt | 1 + test/unit/utOpenGEXImportExport.cpp | 2 - test/unit/utQ3DImportExport.cpp | 61 +++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 test/unit/utQ3DImportExport.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2d065e3be..8627efe14 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -130,6 +130,7 @@ SET( TEST_SRCS unit/utVector3.cpp unit/utXImporterExporter.cpp unit/utD3MFImportExport.cpp + unit/utQ3DImportExport.cpp unit/utProfiler.cpp ) diff --git a/test/unit/utOpenGEXImportExport.cpp b/test/unit/utOpenGEXImportExport.cpp index 60e8ce3a2..021a7bf81 100644 --- a/test/unit/utOpenGEXImportExport.cpp +++ b/test/unit/utOpenGEXImportExport.cpp @@ -54,8 +54,6 @@ public: Assimp::Importer importer; const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/OpenGEX/Example.ogex", 0 ); return nullptr != scene; - - return true; } }; diff --git a/test/unit/utQ3DImportExport.cpp b/test/unit/utQ3DImportExport.cpp new file mode 100644 index 000000000..6d76e001f --- /dev/null +++ b/test/unit/utQ3DImportExport.cpp @@ -0,0 +1,61 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#include "UnitTestPCH.h" + +#include "AbstractImportExportBase.h" + +#include + +using namespace Assimp; + +class utQ3DImportExport : public AbstractImportExportBase { +public: + bool importerTest() override { + Assimp::Importer importer; + const aiScene *scene = importer.ReadFile(ASSIMP_TEST_MODELS_DIR "/Q3D/earth.q3o", 0); + return nullptr != scene; + } +}; + +TEST_F(utQ3DImportExport, importTest) { + EXPECT_TRUE(importerTest()); +} From 82f0dae8350282b6aac1574660ad7c5bfcf4d865 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 31 Aug 2017 11:53:00 +0200 Subject: [PATCH 070/305] FIx build --- test/unit/utQ3DImportExport.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/utQ3DImportExport.cpp b/test/unit/utQ3DImportExport.cpp index 6d76e001f..8f3c52b28 100644 --- a/test/unit/utQ3DImportExport.cpp +++ b/test/unit/utQ3DImportExport.cpp @@ -49,9 +49,9 @@ using namespace Assimp; class utQ3DImportExport : public AbstractImportExportBase { public: - bool importerTest() override { + virtual bool importerTest() { Assimp::Importer importer; - const aiScene *scene = importer.ReadFile(ASSIMP_TEST_MODELS_DIR "/Q3D/earth.q3o", 0); + const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/Q3D/earth.q3o", 0 ); return nullptr != scene; } }; From 2df704edef1de3d17b5ca0edd7764f63d2fe9fd7 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 31 Aug 2017 14:06:56 +0200 Subject: [PATCH 071/305] fix the buil --- test/unit/utQ3DImportExport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/utQ3DImportExport.cpp b/test/unit/utQ3DImportExport.cpp index 8f3c52b28..d8195309f 100644 --- a/test/unit/utQ3DImportExport.cpp +++ b/test/unit/utQ3DImportExport.cpp @@ -57,5 +57,5 @@ public: }; TEST_F(utQ3DImportExport, importTest) { - EXPECT_TRUE(importerTest()); + EXPECT_TRUE( importerTest() ); } From f8146e4da261cca2fe7a00eae22e226f3cdf21cb Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 31 Aug 2017 14:15:23 +0200 Subject: [PATCH 072/305] fix the buil --- test/unit/utD3MFImportExport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/utD3MFImportExport.cpp b/test/unit/utD3MFImportExport.cpp index b77f44f86..0c322420c 100644 --- a/test/unit/utD3MFImportExport.cpp +++ b/test/unit/utD3MFImportExport.cpp @@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. class utD3MFImporterExporter : public AbstractImportExportBase { public: - bool importerTest() override { + virtual bool importerTest() { Assimp::Importer importer; const aiScene *scene = importer.ReadFile(ASSIMP_TEST_MODELS_DIR "/3MF/box.3mf", 0); EXPECT_EQ( 1u, scene->mNumMeshes ); From 3de9bbb73d7fe3bae53b73efecadc781d1d7cccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sat, 2 Sep 2017 19:44:51 +0200 Subject: [PATCH 073/305] fixed spelling error --- code/PlyLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/PlyLoader.cpp b/code/PlyLoader.cpp index 090473893..d348fe113 100644 --- a/code/PlyLoader.cpp +++ b/code/PlyLoader.cpp @@ -538,7 +538,7 @@ void PLYImporter::LoadFace(const PLY::Element* pcElement, const PLY::ElementInst ai_assert(NULL != instElement); if (mGeneratedMesh == NULL) - throw DeadlyImportError("Invalid .ply file: Vertices shoud be declared before faces"); + throw DeadlyImportError("Invalid .ply file: Vertices should be declared before faces"); bool bOne = false; From 042597552acdfa9c837190253cef2c07963f9cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sat, 2 Sep 2017 19:46:57 +0200 Subject: [PATCH 074/305] split setup.py into multiple lines to make it easier for patching in Debian... --- port/PyAssimp/setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/port/PyAssimp/setup.py b/port/PyAssimp/setup.py index e683f7a81..26cf9b400 100644 --- a/port/PyAssimp/setup.py +++ b/port/PyAssimp/setup.py @@ -8,6 +8,9 @@ setup(name='pyassimp', description='Python bindings for the Open Asset Import Library (ASSIMP)', url='https://github.com/assimp/assimp', packages=['pyassimp'], - data_files=[('share/pyassimp', ['README.md']), - ('share/examples/pyassimp', ['scripts/' + f for f in os.listdir('scripts/')])], requires=['numpy'] + data_files=[ + ('share/pyassimp', ['README.md']), + ('share/examples/pyassimp', ['scripts/' + f for f in os.listdir('scripts/')]) + ], + requires=['numpy'] ) From eb0ace4e185794384c4eefa41d8e83902344082b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?IOhannes=20m=20zm=C3=B6lnig?= Date: Sun, 3 Sep 2017 23:03:48 +0200 Subject: [PATCH 075/305] exclude repository-settings from source-package generated via 'git archive' --- .gitattributes | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitattributes b/.gitattributes index 7f9357f62..f3df90d38 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,10 @@ CHANGES text eol=lf CREDITS text eol=lf LICENSE text eol=lf Readme.md text eol=lf +# make sure that repo-specific settings (.gitignore, CI-setup,...) +# are excluded from the source-package generated via 'git archive' +.git* export-ignore +/.travis* export-ignore +/.coveralls* export-ignore +appveyor.yml export-ignore + From c143d2e02cab75cdf5171b6f0dfc42569c061888 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 7 Sep 2017 20:30:17 +0200 Subject: [PATCH 076/305] closes https://github.com/assimp/assimp/issues/1404: set name with merged meshes for output mesh. --- code/3DSConverter.cpp | 9 +++-- code/SceneCombiner.cpp | 16 ++++++-- include/assimp/SceneCombiner.h | 7 +--- test/CMakeLists.txt | 1 + test/unit/utSceneCombiner.cpp | 70 ++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 test/unit/utSceneCombiner.cpp diff --git a/code/3DSConverter.cpp b/code/3DSConverter.cpp index da8c918a7..820c28f90 100644 --- a/code/3DSConverter.cpp +++ b/code/3DSConverter.cpp @@ -56,18 +56,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace Assimp; +static const unsigned int NotSet = 0xcdcdcdcd; + // ------------------------------------------------------------------------------------------------ // Setup final material indices, generae a default material if necessary void Discreet3DSImporter::ReplaceDefaultMaterial() { - // Try to find an existing material that matches the // typical default material setting: // - no textures // - diffuse color (in grey!) // NOTE: This is here to workaround the fact that some // exporters are writing a default material, too. - unsigned int idx = 0xcdcdcdcd; + unsigned int idx( NotSet ); for (unsigned int i = 0; i < mScene->mMaterials.size();++i) { std::string s = mScene->mMaterials[i].mName; @@ -93,7 +94,9 @@ void Discreet3DSImporter::ReplaceDefaultMaterial() } idx = i; } - if (0xcdcdcdcd == idx)idx = (unsigned int)mScene->mMaterials.size(); + if ( NotSet == idx ) { + idx = ( unsigned int )mScene->mMaterials.size(); + } // now iterate through all meshes and through all faces and // find all faces that are using the default material diff --git a/code/SceneCombiner.cpp b/code/SceneCombiner.cpp index 9bcce4275..6fb120325 100644 --- a/code/SceneCombiner.cpp +++ b/code/SceneCombiner.cpp @@ -58,6 +58,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "time.h" #include #include +#include #include #include "ScenePrivate.h" @@ -757,7 +758,7 @@ void SceneCombiner::MergeBones(aiMesh* out,std::vector::const_iterator // ------------------------------------------------------------------------------------------------ // Merge a list of meshes -void SceneCombiner::MergeMeshes(aiMesh** _out,unsigned int /*flags*/, +void SceneCombiner::MergeMeshes(aiMesh** _out, unsigned int /*flags*/, std::vector::const_iterator begin, std::vector::const_iterator end) { @@ -772,8 +773,14 @@ void SceneCombiner::MergeMeshes(aiMesh** _out,unsigned int /*flags*/, aiMesh* out = *_out = new aiMesh(); out->mMaterialIndex = (*begin)->mMaterialIndex; + std::string name; // Find out how much output storage we'll need - for (std::vector::const_iterator it = begin; it != end;++it) { + for (std::vector::const_iterator it = begin; it != end; ++it) { + const char *meshName( (*it)->mName.C_Str() ); + name += std::string( meshName ); + if ( it != end - 1 ) { + name += "."; + } out->mNumVertices += (*it)->mNumVertices; out->mNumFaces += (*it)->mNumFaces; out->mNumBones += (*it)->mNumBones; @@ -781,6 +788,7 @@ void SceneCombiner::MergeMeshes(aiMesh** _out,unsigned int /*flags*/, // combine primitive type flags out->mPrimitiveTypes |= (*it)->mPrimitiveTypes; } + out->mName.Set( name.c_str() ); if (out->mNumVertices) { aiVector3D* pv2; @@ -789,7 +797,7 @@ void SceneCombiner::MergeMeshes(aiMesh** _out,unsigned int /*flags*/, if ((**begin).HasPositions()) { pv2 = out->mVertices = new aiVector3D[out->mNumVertices]; - for (std::vector::const_iterator it = begin; it != end;++it) { + for (std::vector::const_iterator it = begin; it != end; ++it) { if ((*it)->mVertices) { ::memcpy(pv2,(*it)->mVertices,(*it)->mNumVertices*sizeof(aiVector3D)); } @@ -809,7 +817,7 @@ void SceneCombiner::MergeMeshes(aiMesh** _out,unsigned int /*flags*/, pv2 += (*it)->mNumVertices; } } - // copy tangents and bitangents + // copy tangents and bi-tangents if ((**begin).HasTangentsAndBitangents()) { pv2 = out->mTangents = new aiVector3D[out->mNumVertices]; diff --git a/include/assimp/SceneCombiner.h b/include/assimp/SceneCombiner.h index ca4e68c7f..ebb5dda00 100644 --- a/include/assimp/SceneCombiner.h +++ b/include/assimp/SceneCombiner.h @@ -217,10 +217,9 @@ public: static void MergeScenes(aiScene** dest,std::vector& src, unsigned int flags = 0); - // ------------------------------------------------------------------- - /** Merges two or more scenes and attaches all sceenes to a specific - * position in the node graph of the masteer scene. + /** Merges two or more scenes and attaches all scenes to a specific + * position in the node graph of the master scene. * * @param dest Receives a pointer to the destination scene. If the * pointer doesn't point to NULL when the function is called, the @@ -236,7 +235,6 @@ public: std::vector& src, unsigned int flags = 0); - // ------------------------------------------------------------------- /** Merges two or more meshes * @@ -255,7 +253,6 @@ public: std::vector::const_iterator begin, std::vector::const_iterator end); - // ------------------------------------------------------------------- /** Merges two or more bones * diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8627efe14..cb559deb3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -116,6 +116,7 @@ SET( TEST_SRCS unit/utRemoveRedundantMaterials.cpp unit/utRemoveVCProcess.cpp unit/utScenePreprocessor.cpp + unit/utSceneCombiner.cpp unit/utSharedPPData.cpp unit/utStringUtils.cpp unit/utSMDImportExport.cpp diff --git a/test/unit/utSceneCombiner.cpp b/test/unit/utSceneCombiner.cpp new file mode 100644 index 000000000..f0e4d5d90 --- /dev/null +++ b/test/unit/utSceneCombiner.cpp @@ -0,0 +1,70 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#include "UnitTestPCH.h" +#include +#include + +using namespace ::Assimp; + +class utSceneCombiner : public ::testing::Test { + // empty +}; + +TEST_F( utSceneCombiner, MergeMeshes_ValidNames_Test ) { + std::vector merge_list; + aiMesh *mesh1 = new aiMesh; + mesh1->mName.Set( "mesh_1" ); + merge_list.push_back( mesh1 ); + + aiMesh *mesh2 = new aiMesh; + mesh2->mName.Set( "mesh_2" ); + merge_list.push_back( mesh2 ); + + aiMesh *mesh3 = new aiMesh; + mesh3->mName.Set( "mesh_3" ); + merge_list.push_back( mesh3 ); + + aiMesh *out( nullptr ); + SceneCombiner::MergeMeshes( &out, 0, merge_list.begin(), merge_list.end() ); + std::string outName = out->mName.C_Str(); + EXPECT_EQ( "mesh_1.mesh_2.mesh_3", outName ); +} From e4c15575614e1bd4715443cb502a3afc49cbeef1 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 7 Sep 2017 22:02:09 +0200 Subject: [PATCH 077/305] Traivis: make build amtrix smaller. --- .travis.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ffebab3cb..38849dae4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,14 +28,16 @@ env: - secure: "lZ7pHQvl5dpZWzBQAaIMf0wqrvtcZ4wiZKeIZjf83TEsflW8+z0uTpIuN30ZV6Glth/Sq1OhLnTP5+N57fZU/1ebA5twHdvP4bS5CIUUg71/CXQZNl36xeaqvxsG/xRrdpKOsPdjAOsQ9KPTQulsX43XDLS7CasMiLvYOpqKcPc=" - PV=r8e PLATF=linux-x86_64 NDK_HOME=${TRAVIS_BUILD_DIR}/android-ndk-${PV} PATH=${PATH}:${NDK_HOME} matrix: - - LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON - - LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - - LINUX=1 SHARED_BUILD=ON ENABLE_COVERALLS=OFF - - LINUX=1 SHARED_BUILD=OFF ENABLE_COVERALLS=OFF - -compiler: - - gcc - - clang + - os: linux LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON + compiler: gcc + - os: linux LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + compiler: clang + - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + compiler: gcc + - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + compiler: clang + - os: osx + osx_image: xcode8.2 install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From c2baa0f59d06cc670c624c388761dd32ec1b8bee Mon Sep 17 00:00:00 2001 From: Giuseppe Barbieri Date: Fri, 8 Sep 2017 15:46:23 +0200 Subject: [PATCH 078/305] Added a link to pure jvm assimp port --- Readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Readme.md b/Readme.md index 88dc6fbc3..a5c31881a 100644 --- a/Readme.md +++ b/Readme.md @@ -119,6 +119,7 @@ Take a look into the `INSTALL` file. Our build system is CMake, if you used CMak * [Pascal](port/AssimpPascal/Readme.md) * [Javascript (Alpha)](https://github.com/makc/assimp2json) * [Unity 3d Plugin] (https://www.assetstore.unity3d.com/en/#!/content/91777) +* [JVM](https://github.com/kotlin-graphics/assimp) Full jvm port (currently supported obj, ply, stl, ~collada) ### Other tools ### [open3mod](https://github.com/acgessler/open3mod) is a powerful 3D model viewer based on Assimp's import and export abilities. From 2ec46cc1882f5f2ebb4402eee0ff181a7132d169 Mon Sep 17 00:00:00 2001 From: Amit Cirt Date: Sat, 9 Sep 2017 19:04:15 +0300 Subject: [PATCH 079/305] fix name lost in mesh and nodes when load with aiProcess_PreTransformVertices flag --- code/PretransformVertices.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/code/PretransformVertices.cpp b/code/PretransformVertices.cpp index 1339053f7..7bfed4292 100644 --- a/code/PretransformVertices.cpp +++ b/code/PretransformVertices.cpp @@ -160,6 +160,11 @@ void PretransformVertices::CollectData( aiScene* pcScene, aiNode* pcNode, unsign unsigned int& num_ref = num_refs[pcNode->mMeshes[i]]; ai_assert(0 != num_ref); --num_ref; + // Save the name of the last mesh + if (num_ref==0) + { + pcMeshOut->mName = pcMesh->mName; + } if (identity) { // copy positions without modifying them @@ -626,9 +631,10 @@ void PretransformVertices::Execute( aiScene* pScene) // now delete all nodes in the scene and build a new // flat node graph with a root node and some level 1 children + aiNode* newRoot = new aiNode(); + newRoot->mName = pScene->mRootNode->mName; delete pScene->mRootNode; pScene->mRootNode = new aiNode(); - pScene->mRootNode->mName.Set(""); if (1 == pScene->mNumMeshes && !pScene->mNumLights && !pScene->mNumCameras) { @@ -646,7 +652,7 @@ void PretransformVertices::Execute( aiScene* pScene) { aiNode* pcNode = *nodes = new aiNode(); pcNode->mParent = pScene->mRootNode; - pcNode->mName.length = ::ai_snprintf(pcNode->mName.data,MAXLEN,"mesh_%u",i); + pcNode->mName = pScene->mMeshes[i]->mName; // setup mesh indices pcNode->mNumMeshes = 1; From 97b67d5cb5c5ca0ccb9ccf62390dee3dbec1d3ee Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 10 Sep 2017 18:47:24 +1000 Subject: [PATCH 080/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/AssbinExporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/AssbinExporter.cpp b/code/AssbinExporter.cpp index e97f27a8a..4ea261718 100644 --- a/code/AssbinExporter.cpp +++ b/code/AssbinExporter.cpp @@ -325,7 +325,7 @@ inline size_t WriteArray(IOStream * stream, const T* in, unsigned int size) { AssbinChunkWriter chunk( container, ASSBIN_CHUNK_AINODE ); - size_t nb_metadata = (node->mMetaData != NULL ? node->mMetaData->mNumProperties : 0); + unsigned int nb_metadata = (node->mMetaData != NULL ? node->mMetaData->mNumProperties : 0); Write(&chunk,node->mName); Write(&chunk,node->mTransformation); From 98532b45bfab304e579c7aef0b16a6cd00142500 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 10 Sep 2017 19:04:44 +1000 Subject: [PATCH 081/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/ColladaExporter.cpp | 10 +++++----- code/ColladaLoader.cpp | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/code/ColladaExporter.cpp b/code/ColladaExporter.cpp index 745975e5d..06604bc8e 100644 --- a/code/ColladaExporter.cpp +++ b/code/ColladaExporter.cpp @@ -866,8 +866,8 @@ void ColladaExporter::WriteController( size_t pIndex) std::vector bind_poses; bind_poses.reserve(mesh->mNumBones * 16); - for( size_t i = 0; i < mesh->mNumBones; ++i) - for( size_t j = 0; j < 4; ++j) + for(unsigned int i = 0; i < mesh->mNumBones; ++i) + for( unsigned int j = 0; j < 4; ++j) bind_poses.insert(bind_poses.end(), mesh->mBones[i]->mOffsetMatrix[j], mesh->mBones[i]->mOffsetMatrix[j] + 4); WriteFloatArray( idstr + "-skin-bind_poses", FloatType_Mat4x4, (const ai_real*) bind_poses.data(), bind_poses.size() / 16); @@ -924,11 +924,11 @@ void ColladaExporter::WriteController( size_t pIndex) ai_uint weight_index = 0; std::vector joint_weight_indices(2 * joint_weight_indices_length, (ai_int)-1); - for( size_t i = 0; i < mesh->mNumBones; ++i) - for( size_t j = 0; j < mesh->mBones[i]->mNumWeights; ++j) + for( unsigned int i = 0; i < mesh->mNumBones; ++i) + for( unsigned j = 0; j < mesh->mBones[i]->mNumWeights; ++j) { unsigned int vId = mesh->mBones[i]->mWeights[j].mVertexId; - for( size_t k = 0; k < num_influences[vId]; ++k) + for( ai_uint k = 0; k < num_influences[vId]; ++k) { if (joint_weight_indices[2 * (accum_influences[vId] + k)] == -1) { diff --git a/code/ColladaLoader.cpp b/code/ColladaLoader.cpp index c68eb27c9..f7ef903f4 100644 --- a/code/ColladaLoader.cpp +++ b/code/ColladaLoader.cpp @@ -728,7 +728,7 @@ aiMesh* ColladaLoader::CreateMesh( const ColladaParser& pParser, const Collada:: ? aiMorphingMethod_MORPH_RELATIVE : aiMorphingMethod_MORPH_NORMALIZED; dstMesh->mAnimMeshes = new aiAnimMesh*[animMeshes.size()]; - dstMesh->mNumAnimMeshes = animMeshes.size(); + dstMesh->mNumAnimMeshes = static_cast(animMeshes.size()); for (unsigned int i = 0; i < animMeshes.size(); i++) dstMesh->mAnimMeshes[i] = animMeshes.at(i); } @@ -1377,9 +1377,9 @@ void ColladaLoader::CreateAnimation( aiScene* pScene, const ColladaParser& pPars { aiNodeAnim* dstAnim = new aiNodeAnim; dstAnim->mNodeName = nodeName; - dstAnim->mNumPositionKeys = resultTrafos.size(); - dstAnim->mNumRotationKeys= resultTrafos.size(); - dstAnim->mNumScalingKeys = resultTrafos.size(); + dstAnim->mNumPositionKeys = static_cast(resultTrafos.size()); + dstAnim->mNumRotationKeys = static_cast(resultTrafos.size()); + dstAnim->mNumScalingKeys = static_cast(resultTrafos.size()); dstAnim->mPositionKeys = new aiVectorKey[resultTrafos.size()]; dstAnim->mRotationKeys = new aiQuatKey[resultTrafos.size()]; dstAnim->mScalingKeys = new aiVectorKey[resultTrafos.size()]; @@ -1445,11 +1445,11 @@ void ColladaLoader::CreateAnimation( aiScene* pScene, const ColladaParser& pPars ++morphAnimChannelIndex; } - morphAnim->mNumKeys = morphTimeValues.size(); + morphAnim->mNumKeys = static_cast(morphTimeValues.size()); morphAnim->mKeys = new aiMeshMorphKey[morphAnim->mNumKeys]; for (unsigned int key = 0; key < morphAnim->mNumKeys; key++) { - morphAnim->mKeys[key].mNumValuesAndWeights = morphChannels.size(); + morphAnim->mKeys[key].mNumValuesAndWeights = static_cast(morphChannels.size()); morphAnim->mKeys[key].mValues = new unsigned int [morphChannels.size()]; morphAnim->mKeys[key].mWeights = new double [morphChannels.size()]; @@ -1470,13 +1470,13 @@ void ColladaLoader::CreateAnimation( aiScene* pScene, const ColladaParser& pPars { aiAnimation* anim = new aiAnimation; anim->mName.Set( pName); - anim->mNumChannels = anims.size(); + anim->mNumChannels = static_cast(anims.size()); if (anim->mNumChannels > 0) { anim->mChannels = new aiNodeAnim*[anims.size()]; std::copy( anims.begin(), anims.end(), anim->mChannels); } - anim->mNumMorphMeshChannels = morphAnims.size(); + anim->mNumMorphMeshChannels = static_cast(morphAnims.size()); if (anim->mNumMorphMeshChannels > 0) { anim->mMorphMeshChannels = new aiMeshMorphAnim*[anim->mNumMorphMeshChannels]; From b7f1277175236b6ce935b9ca9cc2b210b47905b3 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 10 Sep 2017 19:07:37 +1000 Subject: [PATCH 082/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/ObjFileMtlImporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ObjFileMtlImporter.cpp b/code/ObjFileMtlImporter.cpp index 69f35a67d..36bb6c2cb 100644 --- a/code/ObjFileMtlImporter.cpp +++ b/code/ObjFileMtlImporter.cpp @@ -304,7 +304,7 @@ void ObjFileMtlImporter::createMaterial() m_pModel->m_pCurrentMaterial = new ObjFile::Material(); m_pModel->m_pCurrentMaterial->MaterialName.Set( name ); if (m_pModel->m_pCurrentMesh) { - m_pModel->m_pCurrentMesh->m_uiMaterialIndex = m_pModel->m_MaterialLib.size() - 1; + m_pModel->m_pCurrentMesh->m_uiMaterialIndex = static_cast(m_pModel->m_MaterialLib.size() - 1); } m_pModel->m_MaterialLib.push_back( name ); m_pModel->m_MaterialMap[ name ] = m_pModel->m_pCurrentMaterial; From 6e02bcd8d6b187d10f30c536dbd2e3bf3c3af85d Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 10 Sep 2017 19:10:59 +1000 Subject: [PATCH 083/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/PlyParser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/PlyParser.cpp b/code/PlyParser.cpp index d978b513b..f4b68a83f 100644 --- a/code/PlyParser.cpp +++ b/code/PlyParser.cpp @@ -618,7 +618,7 @@ bool PLY::DOM::ParseInstanceBinary(IOStreamBuffer &streamBuffer, DOM* p_pc } streamBuffer.getNextBlock(buffer); - unsigned int bufferSize = buffer.size(); + unsigned int bufferSize = static_cast(buffer.size()); const char* pCur = (char*)&buffer[0]; if (!p_pcOut->ParseElementInstanceListsBinary(streamBuffer, buffer, pCur, bufferSize, loader, p_bBE)) { @@ -1025,7 +1025,7 @@ bool PLY::PropertyInstance::ParseValueBinary(IOStreamBuffer &streamBuffer, buffer = std::vector(buffer.end() - bufferSize, buffer.end()); buffer.insert(buffer.end(), nbuffer.begin(), nbuffer.end()); nbuffer.clear(); - bufferSize = buffer.size(); + bufferSize = static_cast(buffer.size()); pCur = (char*)&buffer[0]; } else From 6db0a63d6ef5e2b70014d342810955abc6cec088 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 10 Sep 2017 19:13:12 +1000 Subject: [PATCH 084/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/IFCProfile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/IFCProfile.cpp b/code/IFCProfile.cpp index e94d98a04..595159a6b 100644 --- a/code/IFCProfile.cpp +++ b/code/IFCProfile.cpp @@ -128,7 +128,7 @@ void ProcessParametrizedProfile(const IfcParameterizedProfileDef& def, TempMesh& meshout.verts.push_back( IfcVector3( std::cos(angle)*radius, std::sin(angle)*radius, 0.f )); } - meshout.vertcnt.push_back(segments); + meshout.vertcnt.push_back(static_cast(segments)); } else if( const IfcIShapeProfileDef* const ishape = def.ToPtr()) { // construct simplified IBeam shape From 1167edaeca09b07a8ea20036d83d626180d48175 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Mon, 11 Sep 2017 00:09:35 +1000 Subject: [PATCH 085/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/MMDImporter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/MMDImporter.cpp b/code/MMDImporter.cpp index 149761c34..84797cba2 100644 --- a/code/MMDImporter.cpp +++ b/code/MMDImporter.cpp @@ -278,7 +278,7 @@ aiMesh *MMDImporter::CreateMesh(const pmx::PmxModel *pModel, bone_vertex_map[vsBDEF2_ptr->bone_index1].push_back( aiVertexWeight(index, vsBDEF2_ptr->bone_weight)); bone_vertex_map[vsBDEF2_ptr->bone_index2].push_back( - aiVertexWeight(index, 1.0 - vsBDEF2_ptr->bone_weight)); + aiVertexWeight(index, 1.0f - vsBDEF2_ptr->bone_weight)); break; case pmx::PmxVertexSkinningType::BDEF4: bone_vertex_map[vsBDEF4_ptr->bone_index1].push_back( @@ -295,7 +295,7 @@ aiMesh *MMDImporter::CreateMesh(const pmx::PmxModel *pModel, bone_vertex_map[vsSDEF_ptr->bone_index1].push_back( aiVertexWeight(index, vsSDEF_ptr->bone_weight)); bone_vertex_map[vsSDEF_ptr->bone_index2].push_back( - aiVertexWeight(index, 1.0 - vsSDEF_ptr->bone_weight)); + aiVertexWeight(index, 1.0f - vsSDEF_ptr->bone_weight)); break; case pmx::PmxVertexSkinningType::QDEF: const auto vsQDEF_ptr = @@ -325,7 +325,7 @@ aiMesh *MMDImporter::CreateMesh(const pmx::PmxModel *pModel, aiMatrix4x4::Translation(-pos, pBone->mOffsetMatrix); auto it = bone_vertex_map.find(ii); if (it != bone_vertex_map.end()) { - pBone->mNumWeights = it->second.size(); + pBone->mNumWeights = static_cast(it->second.size()); pBone->mWeights = it->second.data(); it->second.swap(*(new vector)); } From e40cd6c13c0d5c85e932c4d958c4c13d8b6d1d20 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Mon, 11 Sep 2017 00:09:40 +1000 Subject: [PATCH 086/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/X3DImporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/X3DImporter.cpp b/code/X3DImporter.cpp index 669f46f95..aaf99b309 100644 --- a/code/X3DImporter.cpp +++ b/code/X3DImporter.cpp @@ -558,7 +558,7 @@ void X3DImporter::XML_ReadNode_GetAttrVal_AsArrF(const int pAttrIdx, std::vector WordIterator wordItBegin(val, val + strlen(val)); WordIterator wordItEnd; - std::transform(wordItBegin, wordItEnd, std::back_inserter(pValue), [](const char *match) { return atof(match); }); + std::transform(wordItBegin, wordItEnd, std::back_inserter(pValue), [](const char *match) { return static_cast(atof(match)); }); } } From 698cd5826d5be44964a24fb32a1c3c92f0d8daac Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Mon, 11 Sep 2017 00:16:26 +1000 Subject: [PATCH 087/305] Fixed warnings when compiling for x64 on MSVC through VS 2017 v15.3.3. --- code/glTF2Asset.inl | 2 +- code/glTF2Exporter.cpp | 10 +++++----- code/glTFExporter.cpp | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 44a1cce00..09e647cca 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -242,7 +242,7 @@ Ref LazyDict::Create(const char* id) } T* inst = new T(); inst->id = id; - inst->index = mObjs.size(); + inst->index = static_cast(mObjs.size()); return Add(inst); } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 11694a89d..7daef7000 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -886,7 +886,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmPositionKeys[frameIndex].mTime / ticksPerSecond; } - Ref timeAccessor = ExportData(mAsset, animId, buffer, numKeyframes, &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); + Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); if (timeAccessor) animRef->Parameters.TIME = timeAccessor; } @@ -899,7 +899,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmPositionKeys[frameIndex].mValue; } - Ref tranAccessor = ExportData(mAsset, animId, buffer, numKeyframes, translationData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + Ref tranAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), translationData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); if ( tranAccessor ) { animRef->Parameters.translation = tranAccessor; } @@ -915,7 +915,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmScalingKeys[frameIndex].mValue; } - Ref scaleAccessor = ExportData(mAsset, animId, buffer, numKeyframes, scaleData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + Ref scaleAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), scaleData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); if ( scaleAccessor ) { animRef->Parameters.scale = scaleAccessor; } @@ -934,7 +934,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmRotationKeys[frameIndex].mValue.w; } - Ref rotAccessor = ExportData(mAsset, animId, buffer, numKeyframes, rotationData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); + Ref rotAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), rotationData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); if ( rotAccessor ) { animRef->Parameters.rotation = rotAccessor; } @@ -989,7 +989,7 @@ void glTF2Exporter::ExportAnimations() Animation::AnimChannel tmpAnimChannel; Animation::AnimSampler tmpAnimSampler; - tmpAnimChannel.sampler = animRef->Samplers.size(); + tmpAnimChannel.sampler = static_cast(animRef->Samplers.size()); tmpAnimChannel.target.path = channelType; tmpAnimSampler.output = channelType; tmpAnimSampler.id = name + "_" + channelType; diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index f3fa5e526..8967a7aa7 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -875,7 +875,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmPositionKeys[frameIndex].mTime / ticksPerSecond; } - Ref timeAccessor = ExportData(mAsset, animId, buffer, numKeyframes, &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); + Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); if (timeAccessor) animRef->Parameters.TIME = timeAccessor; } @@ -888,7 +888,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmPositionKeys[frameIndex].mValue; } - Ref tranAccessor = ExportData(mAsset, animId, buffer, numKeyframes, translationData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + Ref tranAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), translationData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); if ( tranAccessor ) { animRef->Parameters.translation = tranAccessor; } @@ -904,7 +904,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmScalingKeys[frameIndex].mValue; } - Ref scaleAccessor = ExportData(mAsset, animId, buffer, numKeyframes, scaleData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); + Ref scaleAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), scaleData, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); if ( scaleAccessor ) { animRef->Parameters.scale = scaleAccessor; } @@ -923,7 +923,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmRotationKeys[frameIndex].mValue.w; } - Ref rotAccessor = ExportData(mAsset, animId, buffer, numKeyframes, rotationData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); + Ref rotAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), rotationData, AttribType::VEC4, AttribType::VEC4, ComponentType_FLOAT); if ( rotAccessor ) { animRef->Parameters.rotation = rotAccessor; } From 63d3655f1b8b73e3c15799e061baaa8fa95e086f Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 24 Aug 2017 17:15:48 -0400 Subject: [PATCH 088/305] Duplicate gltfImporter as gltf2Importer; Include glTF2 importer in CMake List --- code/CMakeLists.txt | 2 + code/ImporterRegistry.cpp | 4 +- code/glTF2Importer.cpp | 680 ++++++++++++++++++++++++++++++++++++++ code/glTF2Importer.h | 91 +++++ 4 files changed, 775 insertions(+), 2 deletions(-) create mode 100644 code/glTF2Importer.cpp create mode 100644 code/glTF2Importer.h diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 236e6d7ef..0e7aad6cc 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -665,6 +665,8 @@ ADD_ASSIMP_IMPORTER( GLTF glTF2Asset.inl glTF2AssetWriter.h glTF2AssetWriter.inl + glTF2Importer.cpp + glTF2Importer.h glTF2Exporter.h glTF2Exporter.cpp ) diff --git a/code/ImporterRegistry.cpp b/code/ImporterRegistry.cpp index afc308075..2b5d9c1a0 100644 --- a/code/ImporterRegistry.cpp +++ b/code/ImporterRegistry.cpp @@ -181,7 +181,7 @@ corresponding preprocessor flag to selectively disable formats. # include "AssbinLoader.h" #endif #ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER -# include "glTFImporter.h" +# include "glTF2Importer.h" #endif #ifndef ASSIMP_BUILD_NO_C4D_IMPORTER # include "C4DImporter.h" @@ -335,7 +335,7 @@ void GetImporterInstanceList(std::vector< BaseImporter* >& out) out.push_back( new AssbinImporter() ); #endif #if ( !defined ASSIMP_BUILD_NO_GLTF_IMPORTER ) - out.push_back( new glTFImporter() ); + out.push_back( new glTF2Importer() ); #endif #if ( !defined ASSIMP_BUILD_NO_C4D_IMPORTER ) out.push_back( new C4DImporter() ); diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp new file mode 100644 index 000000000..bd4f42600 --- /dev/null +++ b/code/glTF2Importer.cpp @@ -0,0 +1,680 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +#ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER + +#include "glTF2Importer.h" +#include "StringComparison.h" +#include "StringUtils.h" + +#include +#include +#include +#include +#include + +#include + +#include "MakeVerboseFormat.h" + +#include "glTF2Asset.h" +// This is included here so WriteLazyDict's definition is found. +#include "glTF2AssetWriter.h" + +using namespace Assimp; +using namespace glTF2; + + +// +// glTF2Importer +// + +static const aiImporterDesc desc = { + "glTF2 Importer", + "", + "", + "", + aiImporterFlags_SupportTextFlavour | aiImporterFlags_SupportBinaryFlavour | aiImporterFlags_SupportCompressedFlavour + | aiImporterFlags_LimitedSupport | aiImporterFlags_Experimental, + 0, + 0, + 0, + 0, + "gltf glb" +}; + +glTF2Importer::glTF2Importer() +: BaseImporter() +, meshOffsets() +, embeddedTexIdxs() +, mScene( NULL ) { + // empty +} + +glTF2Importer::~glTF2Importer() { + // empty +} + +const aiImporterDesc* glTF2Importer::GetInfo() const +{ + return &desc; +} + +bool glTF2Importer::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool checkSig) const +{ + const std::string& extension = GetExtension(pFile); + + if (extension == "gltf" || extension == "glb") + return true; + + if ((checkSig || !extension.length()) && pIOHandler) { + char buffer[4]; + + std::unique_ptr pStream(pIOHandler->Open(pFile)); + if (pStream && pStream->Read(buffer, sizeof(buffer), 1) == 1) { + if (memcmp(buffer, AI_GLB_MAGIC_NUMBER, sizeof(buffer)) == 0) { + return true; // Has GLB header + } + else if (memcmp(buffer, "{\r\n ", sizeof(buffer)) == 0 + || memcmp(buffer, "{\n ", sizeof(buffer)) == 0) { + // seems a JSON file, and we're the only format that can read them + return true; + } + } + } + + return false; +} + + + +//static void CopyValue(const glTF2::vec3& v, aiColor3D& out) +//{ +// out.r = v[0]; out.g = v[1]; out.b = v[2]; +//} + +static void CopyValue(const glTF2::vec4& v, aiColor4D& out) +{ + out.r = v[0]; out.g = v[1]; out.b = v[2]; out.a = v[3]; +} + +static void CopyValue(const glTF2::vec4& v, aiColor3D& out) +{ + out.r = v[0]; out.g = v[1]; out.b = v[2]; +} + +static void CopyValue(const glTF2::vec3& v, aiVector3D& out) +{ + out.x = v[0]; out.y = v[1]; out.z = v[2]; +} + +static void CopyValue(const glTF2::vec4& v, aiQuaternion& out) +{ + out.x = v[0]; out.y = v[1]; out.z = v[2]; out.w = v[3]; +} + +static void CopyValue(const glTF2::mat4& v, aiMatrix4x4& o) +{ + o.a1 = v[ 0]; o.b1 = v[ 1]; o.c1 = v[ 2]; o.d1 = v[ 3]; + o.a2 = v[ 4]; o.b2 = v[ 5]; o.c2 = v[ 6]; o.d2 = v[ 7]; + o.a3 = v[ 8]; o.b3 = v[ 9]; o.c3 = v[10]; o.d3 = v[11]; + o.a4 = v[12]; o.b4 = v[13]; o.c4 = v[14]; o.d4 = v[15]; +} + +inline void SetMaterialColorProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TexProperty prop, aiMaterial* mat, + aiTextureType texType, const char* pKey, unsigned int type, unsigned int idx) +{ + if (prop.texture) { + if (prop.texture->source) { + aiString uri(prop.texture->source->uri); + + int texIdx = embeddedTexIdxs[prop.texture->source.GetIndex()]; + if (texIdx != -1) { // embedded + // setup texture reference string (copied from ColladaLoader::FindFilenameForEffectTexture) + uri.data[0] = '*'; + uri.length = 1 + ASSIMP_itoa10(uri.data + 1, MAXLEN - 1, texIdx); + } + + mat->AddProperty(&uri, _AI_MATKEY_TEXTURE_BASE, texType, 0); + } + } + else { + aiColor4D col; + CopyValue(prop.color, col); + mat->AddProperty(&col, 1, pKey, type, idx); + } +} + +void glTF2Importer::ImportMaterials(glTF2::Asset& r) +{ + mScene->mNumMaterials = unsigned(r.materials.Size()); + mScene->mMaterials = new aiMaterial*[mScene->mNumMaterials]; + + for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { + aiMaterial* aimat = mScene->mMaterials[i] = new aiMaterial(); + + Material& mat = r.materials[i]; + + /*if (!mat.name.empty())*/ { + aiString str(mat.id /*mat.name*/); + aimat->AddProperty(&str, AI_MATKEY_NAME); + } + + SetMaterialColorProperty(embeddedTexIdxs, r, mat.diffuse, aimat, aiTextureType_DIFFUSE, AI_MATKEY_COLOR_DIFFUSE); + SetMaterialColorProperty(embeddedTexIdxs, r, mat.specular, aimat, aiTextureType_SPECULAR, AI_MATKEY_COLOR_SPECULAR); + SetMaterialColorProperty(embeddedTexIdxs, r, mat.ambient, aimat, aiTextureType_AMBIENT, AI_MATKEY_COLOR_AMBIENT); + + if (mat.shininess > 0.f) { + aimat->AddProperty(&mat.shininess, 1, AI_MATKEY_SHININESS); + } + } +} + + +static inline void SetFace(aiFace& face, int a) +{ + face.mNumIndices = 1; + face.mIndices = new unsigned int[1]; + face.mIndices[0] = a; +} + +static inline void SetFace(aiFace& face, int a, int b) +{ + face.mNumIndices = 2; + face.mIndices = new unsigned int[2]; + face.mIndices[0] = a; + face.mIndices[1] = b; +} + +static inline void SetFace(aiFace& face, int a, int b, int c) +{ + face.mNumIndices = 3; + face.mIndices = new unsigned int[3]; + face.mIndices[0] = a; + face.mIndices[1] = b; + face.mIndices[2] = c; +} + +static inline bool CheckValidFacesIndices(aiFace* faces, unsigned nFaces, unsigned nVerts) +{ + for (unsigned i = 0; i < nFaces; ++i) { + for (unsigned j = 0; j < faces[i].mNumIndices; ++j) { + unsigned idx = faces[i].mIndices[j]; + if (idx >= nVerts) + return false; + } + } + return true; +} + +void glTF2Importer::ImportMeshes(glTF2::Asset& r) +{ + std::vector meshes; + + unsigned int k = 0; + + for (unsigned int m = 0; m < r.meshes.Size(); ++m) { + Mesh& mesh = r.meshes[m]; + + // Check if mesh extensions is used + if(mesh.Extension.size() > 0) + { + for(Mesh::SExtension* cur_ext : mesh.Extension) + { +#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC + if(cur_ext->Type == Mesh::SExtension::EType::Compression_Open3DGC) + { + // Limitations for meshes when using Open3DGC-compression. + // It's a current limitation of sp... Specification have not this part still - about mesh compression. Why only one primitive? + // Because glTF is very flexibly. But in fact it ugly flexible. Every primitive can has own set of accessors and accessors can + // point to a-a-a-a-any part of buffer (through bufferview of course) and even to another buffer. We know that "Open3DGC-compression" + // is applicable only to part of buffer. As we can't guaranty continuity of the data for decoder, we will limit quantity of primitives. + // Yes indices, coordinates etc. still can br stored in different buffers, but with current specification it's a exporter problem. + // Also primitive can has only one of "POSITION", "NORMAL" and less then "AI_MAX_NUMBER_OF_TEXTURECOORDS" of "TEXCOORD". All accessor + // of primitive must point to one continuous region of the buffer. + if(mesh.primitives.size() > 2) throw DeadlyImportError("GLTF: When using Open3DGC compression then only one primitive per mesh are allowed."); + + Mesh::SCompression_Open3DGC* o3dgc_ext = (Mesh::SCompression_Open3DGC*)cur_ext; + Ref buf = r.buffers.Get(o3dgc_ext->Buffer); + + buf->EncodedRegion_SetCurrent(mesh.id); + } + else +#endif + { + throw DeadlyImportError("GLTF: Can not import mesh: unknown mesh extension (code: \"" + to_string(cur_ext->Type) + + "\"), only Open3DGC is supported."); + } + } + }// if(mesh.Extension.size() > 0) + + meshOffsets.push_back(k); + k += unsigned(mesh.primitives.size()); + + for (unsigned int p = 0; p < mesh.primitives.size(); ++p) { + Mesh::Primitive& prim = mesh.primitives[p]; + + aiMesh* aim = new aiMesh(); + meshes.push_back(aim); + + aim->mName = mesh.id; + if (mesh.primitives.size() > 1) { + size_t& len = aim->mName.length; + aim->mName.data[len] = '-'; + len += 1 + ASSIMP_itoa10(aim->mName.data + len + 1, unsigned(MAXLEN - len - 1), p); + } + + switch (prim.mode) { + case PrimitiveMode_POINTS: + aim->mPrimitiveTypes |= aiPrimitiveType_POINT; + break; + + case PrimitiveMode_LINES: + case PrimitiveMode_LINE_LOOP: + case PrimitiveMode_LINE_STRIP: + aim->mPrimitiveTypes |= aiPrimitiveType_LINE; + break; + + case PrimitiveMode_TRIANGLES: + case PrimitiveMode_TRIANGLE_STRIP: + case PrimitiveMode_TRIANGLE_FAN: + aim->mPrimitiveTypes |= aiPrimitiveType_TRIANGLE; + break; + } + + Mesh::Primitive::Attributes& attr = prim.attributes; + + if (attr.position.size() > 0 && attr.position[0]) { + aim->mNumVertices = attr.position[0]->count; + attr.position[0]->ExtractData(aim->mVertices); + } + + if (attr.normal.size() > 0 && attr.normal[0]) attr.normal[0]->ExtractData(aim->mNormals); + + for (size_t tc = 0; tc < attr.texcoord.size() && tc < AI_MAX_NUMBER_OF_TEXTURECOORDS; ++tc) { + attr.texcoord[tc]->ExtractData(aim->mTextureCoords[tc]); + aim->mNumUVComponents[tc] = attr.texcoord[tc]->GetNumComponents(); + + aiVector3D* values = aim->mTextureCoords[tc]; + for (unsigned int i = 0; i < aim->mNumVertices; ++i) { + values[i].y = 1 - values[i].y; // Flip Y coords + } + } + + + if (prim.indices) { + aiFace* faces = 0; + unsigned int nFaces = 0; + + unsigned int count = prim.indices->count; + + Accessor::Indexer data = prim.indices->GetIndexer(); + ai_assert(data.IsValid()); + + switch (prim.mode) { + case PrimitiveMode_POINTS: { + nFaces = count; + faces = new aiFace[nFaces]; + for (unsigned int i = 0; i < count; ++i) { + SetFace(faces[i], data.GetUInt(i)); + } + break; + } + + case PrimitiveMode_LINES: { + nFaces = count / 2; + faces = new aiFace[nFaces]; + for (unsigned int i = 0; i < count; i += 2) { + SetFace(faces[i / 2], data.GetUInt(i), data.GetUInt(i + 1)); + } + break; + } + + case PrimitiveMode_LINE_LOOP: + case PrimitiveMode_LINE_STRIP: { + nFaces = count - ((prim.mode == PrimitiveMode_LINE_STRIP) ? 1 : 0); + faces = new aiFace[nFaces]; + SetFace(faces[0], data.GetUInt(0), data.GetUInt(1)); + for (unsigned int i = 2; i < count; ++i) { + SetFace(faces[i - 1], faces[i - 2].mIndices[1], data.GetUInt(i)); + } + if (prim.mode == PrimitiveMode_LINE_LOOP) { // close the loop + SetFace(faces[count - 1], faces[count - 2].mIndices[1], faces[0].mIndices[0]); + } + break; + } + + case PrimitiveMode_TRIANGLES: { + nFaces = count / 3; + faces = new aiFace[nFaces]; + for (unsigned int i = 0; i < count; i += 3) { + SetFace(faces[i / 3], data.GetUInt(i), data.GetUInt(i + 1), data.GetUInt(i + 2)); + } + break; + } + case PrimitiveMode_TRIANGLE_STRIP: { + nFaces = count - 2; + faces = new aiFace[nFaces]; + SetFace(faces[0], data.GetUInt(0), data.GetUInt(1), data.GetUInt(2)); + for (unsigned int i = 3; i < count; ++i) { + SetFace(faces[i - 2], faces[i - 1].mIndices[1], faces[i - 1].mIndices[2], data.GetUInt(i)); + } + break; + } + case PrimitiveMode_TRIANGLE_FAN: + nFaces = count - 2; + faces = new aiFace[nFaces]; + SetFace(faces[0], data.GetUInt(0), data.GetUInt(1), data.GetUInt(2)); + for (unsigned int i = 3; i < count; ++i) { + SetFace(faces[i - 2], faces[0].mIndices[0], faces[i - 1].mIndices[2], data.GetUInt(i)); + } + break; + } + + if (faces) { + aim->mFaces = faces; + aim->mNumFaces = nFaces; + ai_assert(CheckValidFacesIndices(faces, nFaces, aim->mNumVertices)); + } + } + + + if (prim.material) { + aim->mMaterialIndex = prim.material.GetIndex(); + } + } + } + + meshOffsets.push_back(k); + + CopyVector(meshes, mScene->mMeshes, mScene->mNumMeshes); +} + +void glTF2Importer::ImportCameras(glTF2::Asset& r) +{ + if (!r.cameras.Size()) return; + + mScene->mNumCameras = r.cameras.Size(); + mScene->mCameras = new aiCamera*[r.cameras.Size()]; + + for (size_t i = 0; i < r.cameras.Size(); ++i) { + Camera& cam = r.cameras[i]; + + aiCamera* aicam = mScene->mCameras[i] = new aiCamera(); + + if (cam.type == Camera::Perspective) { + + aicam->mAspect = cam.cameraProperties.perspective.aspectRatio; + aicam->mHorizontalFOV = cam.cameraProperties.perspective.yfov * aicam->mAspect; + aicam->mClipPlaneFar = cam.cameraProperties.perspective.zfar; + aicam->mClipPlaneNear = cam.cameraProperties.perspective.znear; + } + else { + // assimp does not support orthographic cameras + } + } +} + +void glTF2Importer::ImportLights(glTF2::Asset& r) +{ + if (!r.lights.Size()) return; + + mScene->mNumLights = r.lights.Size(); + mScene->mLights = new aiLight*[r.lights.Size()]; + + for (size_t i = 0; i < r.lights.Size(); ++i) { + Light& l = r.lights[i]; + + aiLight* ail = mScene->mLights[i] = new aiLight(); + + switch (l.type) { + case Light::Type_directional: + ail->mType = aiLightSource_DIRECTIONAL; break; + + case Light::Type_spot: + ail->mType = aiLightSource_SPOT; break; + + case Light::Type_ambient: + ail->mType = aiLightSource_AMBIENT; break; + + default: // Light::Type_point + ail->mType = aiLightSource_POINT; break; + } + + CopyValue(l.color, ail->mColorAmbient); + CopyValue(l.color, ail->mColorDiffuse); + CopyValue(l.color, ail->mColorSpecular); + + ail->mAngleOuterCone = l.falloffAngle; + ail->mAngleInnerCone = l.falloffExponent; // TODO fix this, it does not look right at all + + ail->mAttenuationConstant = l.constantAttenuation; + ail->mAttenuationLinear = l.linearAttenuation; + ail->mAttenuationQuadratic = l.quadraticAttenuation; + } +} + + +aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& meshOffsets, glTF2::Ref& ptr) +{ + Node& node = *ptr; + + aiNode* ainode = new aiNode(node.id); + + if (!node.children.empty()) { + ainode->mNumChildren = unsigned(node.children.size()); + ainode->mChildren = new aiNode*[ainode->mNumChildren]; + + for (unsigned int i = 0; i < ainode->mNumChildren; ++i) { + aiNode* child = ImportNode(pScene, r, meshOffsets, node.children[i]); + child->mParent = ainode; + ainode->mChildren[i] = child; + } + } + + aiMatrix4x4& matrix = ainode->mTransformation; + if (node.matrix.isPresent) { + CopyValue(node.matrix.value, matrix); + } + else { + if (node.translation.isPresent) { + aiVector3D trans; + CopyValue(node.translation.value, trans); + aiMatrix4x4 t; + aiMatrix4x4::Translation(trans, t); + matrix = t * matrix; + } + + if (node.scale.isPresent) { + aiVector3D scal(1.f); + CopyValue(node.scale.value, scal); + aiMatrix4x4 s; + aiMatrix4x4::Scaling(scal, s); + matrix = s * matrix; + } + + + if (node.rotation.isPresent) { + aiQuaternion rot; + CopyValue(node.rotation.value, rot); + matrix = aiMatrix4x4(rot.GetMatrix()) * matrix; + } + } + + if (!node.meshes.empty()) { + int count = 0; + for (size_t i = 0; i < node.meshes.size(); ++i) { + int idx = node.meshes[i].GetIndex(); + count += meshOffsets[idx + 1] - meshOffsets[idx]; + } + + ainode->mNumMeshes = count; + ainode->mMeshes = new unsigned int[count]; + + int k = 0; + for (size_t i = 0; i < node.meshes.size(); ++i) { + int idx = node.meshes[i].GetIndex(); + for (unsigned int j = meshOffsets[idx]; j < meshOffsets[idx + 1]; ++j, ++k) { + ainode->mMeshes[k] = j; + } + } + } + + if (node.camera) { + pScene->mCameras[node.camera.GetIndex()]->mName = ainode->mName; + } + + if (node.light) { + pScene->mLights[node.light.GetIndex()]->mName = ainode->mName; + } + + return ainode; +} + +void glTF2Importer::ImportNodes(glTF2::Asset& r) +{ + if (!r.scene) return; + + std::vector< Ref > rootNodes = r.scene->nodes; + + // The root nodes + unsigned int numRootNodes = unsigned(rootNodes.size()); + if (numRootNodes == 1) { // a single root node: use it + mScene->mRootNode = ImportNode(mScene, r, meshOffsets, rootNodes[0]); + } + else if (numRootNodes > 1) { // more than one root node: create a fake root + aiNode* root = new aiNode("ROOT"); + root->mChildren = new aiNode*[numRootNodes]; + for (unsigned int i = 0; i < numRootNodes; ++i) { + aiNode* node = ImportNode(mScene, r, meshOffsets, rootNodes[i]); + node->mParent = root; + root->mChildren[root->mNumChildren++] = node; + } + mScene->mRootNode = root; + } + + //if (!mScene->mRootNode) { + // mScene->mRootNode = new aiNode("EMPTY"); + //} +} + +void glTF2Importer::ImportEmbeddedTextures(glTF2::Asset& r) +{ + embeddedTexIdxs.resize(r.images.Size(), -1); + + int numEmbeddedTexs = 0; + for (size_t i = 0; i < r.images.Size(); ++i) { + if (r.images[i].HasData()) + numEmbeddedTexs += 1; + } + + if (numEmbeddedTexs == 0) + return; + + mScene->mTextures = new aiTexture*[numEmbeddedTexs]; + + // Add the embedded textures + for (size_t i = 0; i < r.images.Size(); ++i) { + Image img = r.images[i]; + if (!img.HasData()) continue; + + int idx = mScene->mNumTextures++; + embeddedTexIdxs[i] = idx; + + aiTexture* tex = mScene->mTextures[idx] = new aiTexture(); + + size_t length = img.GetDataLength(); + void* data = img.StealData(); + + tex->mWidth = static_cast(length); + tex->mHeight = 0; + tex->pcData = reinterpret_cast(data); + + if (!img.mimeType.empty()) { + const char* ext = strchr(img.mimeType.c_str(), '/') + 1; + if (ext) { + if (strcmp(ext, "jpeg") == 0) ext = "jpg"; + + size_t len = strlen(ext); + if (len <= 3) { + strcpy(tex->achFormatHint, ext); + } + } + } + } +} + +void glTF2Importer::InternReadFile(const std::string& pFile, aiScene* pScene, IOSystem* pIOHandler) { + + this->mScene = pScene; + + // read the asset file + glTF2::Asset asset(pIOHandler); + asset.Load(pFile, GetExtension(pFile) == "glb"); + + + // + // Copy the data out + // + + ImportEmbeddedTextures(asset); + ImportMaterials(asset); + + ImportMeshes(asset); + + ImportCameras(asset); + ImportLights(asset); + + ImportNodes(asset); + + // TODO: it does not split the loaded vertices, should it? + //pScene->mFlags |= AI_SCENE_FLAGS_NON_VERBOSE_FORMAT; + MakeVerboseFormatProcess process; + process.Execute(pScene); + + + if (pScene->mNumMeshes == 0) { + pScene->mFlags |= AI_SCENE_FLAGS_INCOMPLETE; + } +} + +#endif // ASSIMP_BUILD_NO_GLTF_IMPORTER + diff --git a/code/glTF2Importer.h b/code/glTF2Importer.h new file mode 100644 index 000000000..6be7131d1 --- /dev/null +++ b/code/glTF2Importer.h @@ -0,0 +1,91 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ +#ifndef AI_GLTF2IMPORTER_H_INC +#define AI_GLTF2IMPORTER_H_INC + +#include "BaseImporter.h" +#include + +struct aiNode; + + +namespace glTF2 +{ + class Asset; +} + +namespace Assimp { + +/** + * Load the glTF2 format. + * https://github.com/KhronosGroup/glTF/tree/master/specification + */ +class glTF2Importer : public BaseImporter{ +public: + glTF2Importer(); + virtual ~glTF2Importer(); + virtual bool CanRead( const std::string& pFile, IOSystem* pIOHandler, bool checkSig ) const; + +protected: + virtual const aiImporterDesc* GetInfo() const; + virtual void InternReadFile( const std::string& pFile, aiScene* pScene, IOSystem* pIOHandler ); + +private: + + std::vector meshOffsets; + + std::vector embeddedTexIdxs; + + aiScene* mScene; + + void ImportEmbeddedTextures(glTF2::Asset& a); + void ImportMaterials(glTF2::Asset& a); + void ImportMeshes(glTF2::Asset& a); + void ImportCameras(glTF2::Asset& a); + void ImportLights(glTF2::Asset& a); + void ImportNodes(glTF2::Asset& a); + +}; + +} // Namespace assimp + +#endif // AI_GLTF2IMPORTER_H_INC + From f814acf33a553b38900b7bd2591d2864aeacdb34 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 24 Aug 2017 17:31:33 -0400 Subject: [PATCH 089/305] Update glTF2 Asset to use indexes --- code/glTF2Asset.h | 6 +- code/glTF2Asset.inl | 309 +++++++++++--------------------------------- 2 files changed, 76 insertions(+), 239 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 52bc18268..2e7a694d8 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -232,7 +232,7 @@ namespace glTF2 case ComponentType_UNSIGNED_BYTE: return 1; default: - throw DeadlyImportError("GLTF: Unsupported Component Type "+t); + throw DeadlyImportError("GLTF: Unsupported Component Type " + std::to_string(t)); } } @@ -629,7 +629,7 @@ namespace glTF2 float zfar; //! The floating-point distance to the far clipping plane. (required) float znear; //! The floating-point distance to the near clipping plane. (required) } ortographic; - }; + } cameraProperties; Camera() {} void Read(Value& obj, Asset& r); @@ -1146,7 +1146,7 @@ namespace glTF2 LazyDict samplers; LazyDict scenes; //LazyDict shaders; - LazyDict skins; + LazyDict skins; //LazyDict techniques; LazyDict textures; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 09e647cca..268deb8a8 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -128,6 +128,12 @@ namespace { return (it != val.MemberEnd() && it->value.IsString()) ? &it->value : 0; } + inline Value* FindUInt(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd() && it->value.IsUint()) ? &it->value : 0; + } + inline Value* FindArray(Value& val, const char* id) { Value::MemberIterator it = val.FindMember(id); @@ -176,7 +182,7 @@ inline void LazyDict::AttachToDocument(Document& doc) } if (container) { - mDict = FindObject(*container, mDictId); + mDict = FindArray(*container, mDictId); } } @@ -189,13 +195,8 @@ inline void LazyDict::DetachFromDocument() template Ref LazyDict::Get(unsigned int i) { - return Ref(mObjs, i); -} -template -Ref LazyDict::Get(const char* id) -{ - id = T::TranslateId(mAsset, id); + std::string id = std::string(mDictId) + "[" + std::to_string(i) + "]"; typename Dict::iterator it = mObjsById.find(id); if (it != mObjsById.end()) { // already created? @@ -207,19 +208,21 @@ Ref LazyDict::Get(const char* id) throw DeadlyImportError("GLTF: Missing section \"" + std::string(mDictId) + "\""); } - Value::MemberIterator obj = mDict->FindMember(id); - if (obj == mDict->MemberEnd()) { - throw DeadlyImportError("GLTF: Missing object with id \"" + std::string(id) + "\" in \"" + mDictId + "\""); + if (!mDict->IsArray()) { + throw DeadlyImportError("GLTF: Field is not an array \"" + std::string(mDictId) + "\""); } - if (!obj->value.IsObject()) { - throw DeadlyImportError("GLTF: Object with id \"" + std::string(id) + "\" is not a JSON object"); + + Value& obj = mDict->operator[](i); + + if (!obj.IsObject()) { + throw DeadlyImportError("GLTF: Object at index \"" + std::to_string(i) + "\" is not a JSON object"); } - // create an instance of the given type T* inst = new T(); inst->id = id; - ReadMember(obj->value, "name", inst->name); - inst->Read(obj->value, mAsset); + ReadMember(obj, "name", inst->name); + inst->Read(obj, mAsset); + return Add(inst); } @@ -437,9 +440,9 @@ inline void Buffer::Grow(size_t amount) inline void BufferView::Read(Value& obj, Asset& r) { - const char* bufferId = MemberOrDefault(obj, "buffer", 0); - if (bufferId) { - buffer = r.buffers.Get(bufferId); + + if (Value* bufferVal = FindUInt(obj, "buffer")) { + buffer = r.buffers.Get(bufferVal->GetUint()); } byteOffset = MemberOrDefault(obj, "byteOffset", 0u); @@ -452,9 +455,9 @@ inline void BufferView::Read(Value& obj, Asset& r) inline void Accessor::Read(Value& obj, Asset& r) { - const char* bufferViewId = MemberOrDefault(obj, "bufferView", 0); - if (bufferViewId) { - bufferView = r.bufferViews.Get(bufferViewId); + + if (Value* bufferViewVal = FindUInt(obj, "bufferView")) { + bufferView = r.bufferViews.Get(bufferViewVal->GetUint()); } byteOffset = MemberOrDefault(obj, "byteOffset", 0u); @@ -611,9 +614,8 @@ inline void Image::Read(Value& obj, Asset& r) ReadMember(*ext, "mimeType", mimeType); - const char* bufferViewId; - if (ReadMember(*ext, "bufferView", bufferViewId)) { - Ref bv = r.bufferViews.Get(bufferViewId); + if (Value* bufferViewVal = FindUInt(*ext, "bufferView")) { + Ref bv = r.bufferViews.Get(bufferViewVal->GetUint()); if (bv) { mDataLength = bv->byteLength; mData = new uint8_t[mDataLength]; @@ -687,23 +689,22 @@ inline void Sampler::SetDefaults() inline void Texture::Read(Value& obj, Asset& r) { - const char* sourcestr; - if (ReadMember(obj, "source", sourcestr)) { - source = r.images.Get(sourcestr); + if (Value* sourceVal = FindUInt(obj, "source")) { + source = r.images.Get(sourceVal->GetUint()); } - const char* samplerstr; - if (ReadMember(obj, "sampler", samplerstr)) { - sampler = r.samplers.Get(samplerstr); + if (Value* samplerVal = FindUInt(obj, "sampler")) { + sampler = r.samplers.Get(samplerVal->GetUint()); } } namespace { inline void ReadMaterialProperty(Asset& r, Value& vals, const char* propName, TexProperty& out) { + //@TODO: update this format if (Value* prop = FindMember(vals, propName)) { - if (prop->IsString()) { - out.texture = r.textures.Get(prop->GetString()); + if (prop->IsUint()) { + out.texture = r.textures.Get(prop->GetUint()); } else { ReadValue(*prop, out.color); @@ -824,22 +825,23 @@ inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) // Valid attribute semantics include POSITION, NORMAL, TEXCOORD, COLOR, JOINT, JOINTMATRIX, // and WEIGHT.Attribute semantics can be of the form[semantic]_[set_index], e.g., TEXCOORD_0, TEXCOORD_1, etc. + //@TODO: update this int undPos = 0; Mesh::AccessorList* vec = 0; if (GetAttribVector(prim, attr, vec, undPos)) { size_t idx = (attr[undPos] == '_') ? atoi(attr + undPos + 1) : 0; if ((*vec).size() <= idx) (*vec).resize(idx + 1); - (*vec)[idx] = pAsset_Root.accessors.Get(it->value.GetString()); + (*vec)[idx] = pAsset_Root.accessors.Get(it->value.GetUint()); } } } - if (Value* indices = FindString(primitive, "indices")) { - prim.indices = pAsset_Root.accessors.Get(indices->GetString()); + if (Value* indices = FindUInt(primitive, "indices")) { + prim.indices = pAsset_Root.accessors.Get(indices->GetUint()); } - if (Value* material = FindString(primitive, "material")) { - prim.material = pAsset_Root.materials.Get(material->GetString()); + if (Value* material = FindUInt(primitive, "material")) { + prim.material = pAsset_Root.materials.Get(material->GetUint()); } } } @@ -914,177 +916,6 @@ mr_skip_extensions: return;// After label some operators must be present. } -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC -inline void Mesh::Decode_O3DGC(const SCompression_Open3DGC& pCompression_Open3DGC, Asset& pAsset_Root) -{ -typedef unsigned short IndicesType;///< \sa glTFExporter::ExportMeshes. - -o3dgc::SC3DMCDecoder decoder; -o3dgc::IndexedFaceSet ifs; -o3dgc::BinaryStream bstream; -uint8_t* decoded_data; -size_t decoded_data_size = 0; -Ref buf = pAsset_Root.buffers.Get(pCompression_Open3DGC.Buffer); - - // Read data from buffer and place it in BinaryStream for decoder. - // Just "Count" because always is used type equivalent to uint8_t. - bstream.LoadFromBuffer(&buf->GetPointer()[pCompression_Open3DGC.Offset], static_cast(pCompression_Open3DGC.Count)); - - // After decoding header we can get size of primitives. - if(decoder.DecodeHeader(ifs, bstream) != o3dgc::O3DGC_OK) throw DeadlyImportError("GLTF: can not decode Open3DGC header."); - - /****************** Get sizes of arrays and check sizes ******************/ - // Note. See "Limitations for meshes when using Open3DGC-compression". - - // Indices - size_t size_coordindex = ifs.GetNCoordIndex() * 3;// See float attributes note. - - if(primitives[0].indices->count != size_coordindex) - throw DeadlyImportError("GLTF: Open3DGC. Compressed indices count (" + std::to_string(size_coordindex) + - ") not equal to uncompressed (" + std::to_string(primitives[0].indices->count) + ")."); - - size_coordindex *= sizeof(IndicesType); - // Coordinates - size_t size_coord = ifs.GetNCoord();// See float attributes note. - - if(primitives[0].attributes.position[0]->count != size_coord) - throw DeadlyImportError("GLTF: Open3DGC. Compressed positions count (" + std::to_string(size_coord) + - ") not equal to uncompressed (" + std::to_string(primitives[0].attributes.position[0]->count) + ")."); - - size_coord *= 3 * sizeof(float); - // Normals - size_t size_normal = ifs.GetNNormal();// See float attributes note. - - if(primitives[0].attributes.normal[0]->count != size_normal) - throw DeadlyImportError("GLTF: Open3DGC. Compressed normals count (" + std::to_string(size_normal) + - ") not equal to uncompressed (" + std::to_string(primitives[0].attributes.normal[0]->count) + ")."); - - size_normal *= 3 * sizeof(float); - // Additional attributes. - std::vector size_floatattr; - std::vector size_intattr; - - size_floatattr.resize(ifs.GetNumFloatAttributes()); - size_intattr.resize(ifs.GetNumIntAttributes()); - - decoded_data_size = size_coordindex + size_coord + size_normal; - for(size_t idx = 0, idx_end = size_floatattr.size(), idx_texcoord = 0; idx < idx_end; idx++) - { - // size = number_of_elements * components_per_element * size_of_component. - // Note. But as you can see above, at first we are use this variable in meaning "count". After checking count of objects... - size_t tval = ifs.GetNFloatAttribute(static_cast(idx)); - - switch(ifs.GetFloatAttributeType(static_cast(idx))) - { - case o3dgc::O3DGC_IFS_FLOAT_ATTRIBUTE_TYPE_TEXCOORD: - // Check situation when encoded data contain texture coordinates but primitive not. - if(idx_texcoord < primitives[0].attributes.texcoord.size()) - { - if(primitives[0].attributes.texcoord[idx]->count != tval) - throw DeadlyImportError("GLTF: Open3DGC. Compressed texture coordinates count (" + std::to_string(tval) + - ") not equal to uncompressed (" + std::to_string(primitives[0].attributes.texcoord[idx]->count) + ")."); - - idx_texcoord++; - } - else - { - ifs.SetNFloatAttribute(static_cast(idx), 0ul);// Disable decoding this attribute. - } - - break; - default: - throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of float attribute: " + to_string(ifs.GetFloatAttributeType(static_cast(idx)))); - } - - tval *= ifs.GetFloatAttributeDim(static_cast(idx)) * sizeof(o3dgc::Real);// After checking count of objects we can get size of array. - size_floatattr[idx] = tval; - decoded_data_size += tval; - } - - for(size_t idx = 0, idx_end = size_intattr.size(); idx < idx_end; idx++) - { - // size = number_of_elements * components_per_element * size_of_component. See float attributes note. - size_t tval = ifs.GetNIntAttribute(static_cast(idx)); - switch( ifs.GetIntAttributeType(static_cast(idx) ) ) - { - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_UNKOWN: - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX: - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_JOINT_ID: - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX_BUFFER_ID: - break; - - default: - throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of int attribute: " + to_string(ifs.GetIntAttributeType(static_cast(idx)))); - } - - tval *= ifs.GetIntAttributeDim(static_cast(idx)) * sizeof(long);// See float attributes note. - size_intattr[idx] = tval; - decoded_data_size += tval; - } - - // Create array for decoded data. - decoded_data = new uint8_t[decoded_data_size]; - - /****************** Set right array regions for decoder ******************/ - - auto get_buf_offset = [](Ref& pAccessor) -> size_t { return pAccessor->byteOffset + pAccessor->bufferView->byteOffset; }; - - // Indices - ifs.SetCoordIndex((IndicesType* const)(decoded_data + get_buf_offset(primitives[0].indices))); - // Coordinates - ifs.SetCoord((o3dgc::Real* const)(decoded_data + get_buf_offset(primitives[0].attributes.position[0]))); - // Normals - if(size_normal) - { - ifs.SetNormal((o3dgc::Real* const)(decoded_data + get_buf_offset(primitives[0].attributes.normal[0]))); - } - - for(size_t idx = 0, idx_end = size_floatattr.size(), idx_texcoord = 0; idx < idx_end; idx++) - { - switch(ifs.GetFloatAttributeType(static_cast(idx))) - { - case o3dgc::O3DGC_IFS_FLOAT_ATTRIBUTE_TYPE_TEXCOORD: - if(idx_texcoord < primitives[0].attributes.texcoord.size()) - { - // See above about absent attributes. - ifs.SetFloatAttribute(static_cast(idx), (o3dgc::Real* const)(decoded_data + get_buf_offset(primitives[0].attributes.texcoord[idx]))); - idx_texcoord++; - } - - break; - default: - throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of float attribute: " + to_string(ifs.GetFloatAttributeType(static_cast(idx)))); - } - } - - for(size_t idx = 0, idx_end = size_intattr.size(); idx < idx_end; idx++) { - switch(ifs.GetIntAttributeType(static_cast(idx))) { - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_UNKOWN: - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX: - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_JOINT_ID: - case o3dgc::O3DGC_IFS_INT_ATTRIBUTE_TYPE_INDEX_BUFFER_ID: - break; - - // ifs.SetIntAttribute(idx, (long* const)(decoded_data + get_buf_offset(primitives[0].attributes.joint))); - default: - throw DeadlyImportError("GLTF: Open3DGC. Unsupported type of int attribute: " + to_string(ifs.GetIntAttributeType(static_cast(idx)))); - } - } - - // - // Decode data - // - if ( decoder.DecodePayload( ifs, bstream ) != o3dgc::O3DGC_OK ) { - throw DeadlyImportError( "GLTF: can not decode Open3DGC data." ); - } - - // Set encoded region for "buffer". - buf->EncodedRegion_Mark(pCompression_Open3DGC.Offset, pCompression_Open3DGC.Count, decoded_data, decoded_data_size, id); - // No. Do not delete "output_data". After calling "EncodedRegion_Mark" bufferView is owner of "output_data". - // "delete [] output_data;" -} -#endif - inline void Camera::Read(Value& obj, Asset& r) { type = MemberOrDefault(obj, "type", Camera::Perspective); @@ -1095,16 +926,16 @@ inline void Camera::Read(Value& obj, Asset& r) if (!it) throw DeadlyImportError("GLTF: Camera missing its parameters"); if (type == Camera::Perspective) { - perspective.aspectRatio = MemberOrDefault(*it, "aspectRatio", 0.f); - perspective.yfov = MemberOrDefault(*it, "yfov", 3.1415f/2.f); - perspective.zfar = MemberOrDefault(*it, "zfar", 100.f); - perspective.znear = MemberOrDefault(*it, "znear", 0.01f); + cameraProperties.perspective.aspectRatio = MemberOrDefault(*it, "aspectRatio", 0.f); + cameraProperties.perspective.yfov = MemberOrDefault(*it, "yfov", 3.1415f/2.f); + cameraProperties.perspective.zfar = MemberOrDefault(*it, "zfar", 100.f); + cameraProperties.perspective.znear = MemberOrDefault(*it, "znear", 0.01f); } else { - ortographic.xmag = MemberOrDefault(obj, "xmag", 1.f); - ortographic.ymag = MemberOrDefault(obj, "ymag", 1.f); - ortographic.zfar = MemberOrDefault(obj, "zfar", 100.f); - ortographic.znear = MemberOrDefault(obj, "znear", 0.01f); + cameraProperties.ortographic.xmag = MemberOrDefault(obj, "xmag", 1.f); + cameraProperties.ortographic.ymag = MemberOrDefault(obj, "ymag", 1.f); + cameraProperties.ortographic.zfar = MemberOrDefault(obj, "zfar", 100.f); + cameraProperties.ortographic.znear = MemberOrDefault(obj, "znear", 0.01f); } } @@ -1156,13 +987,14 @@ inline void Light::SetDefaults() inline void Node::Read(Value& obj, Asset& r) { + if (Value* children = FindArray(obj, "children")) { this->children.reserve(children->Size()); for (unsigned int i = 0; i < children->Size(); ++i) { Value& child = (*children)[i]; - if (child.IsString()) { + if (child.IsNumber()) { // get/create the child node - Ref chn = r.nodes.Get(child.GetString()); + Ref chn = r.nodes.Get(child.GetUint()); if (chn) this->children.push_back(chn); } } @@ -1185,15 +1017,15 @@ inline void Node::Read(Value& obj, Asset& r) this->meshes.reserve(numMeshes); for (unsigned i = 0; i < numMeshes; ++i) { - if ((*meshes)[i].IsString()) { - Ref mesh = r.meshes.Get((*meshes)[i].GetString()); + if ((*meshes)[i].IsNumber()) { + Ref mesh = r.meshes.Get((*meshes)[i].GetUint()); if (mesh) this->meshes.push_back(mesh); } } } - if (Value* camera = FindString(obj, "camera")) { - this->camera = r.cameras.Get(camera->GetString()); + if (Value* camera = FindUInt(obj, "camera")) { + this->camera = r.cameras.Get(camera->GetUint()); if (this->camera) this->camera->id = this->id; } @@ -1204,8 +1036,8 @@ inline void Node::Read(Value& obj, Asset& r) if (r.extensionsUsed.KHR_materials_common) { if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { - if (Value* light = FindString(*ext, "light")) { - this->light = r.lights.Get(light->GetString()); + if (Value* light = FindUInt(*ext, "light")) { + this->light = r.lights.Get(light->GetUint()); } } @@ -1217,8 +1049,8 @@ inline void Scene::Read(Value& obj, Asset& r) { if (Value* array = FindArray(obj, "nodes")) { for (unsigned int i = 0; i < array->Size(); ++i) { - if (!(*array)[i].IsString()) continue; - Ref node = r.nodes.Get((*array)[i].GetString()); + if (!(*array)[i].IsNumber()) continue; + Ref node = r.nodes.Get((*array)[i].GetUint()); if (node) this->nodes.push_back(node); } @@ -1229,13 +1061,14 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { // read the version, etc. - float statedVersion = 0; + std::string statedVersion; + if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); premultipliedAlpha = MemberOrDefault(*obj, "premultipliedAlpha", false); - statedVersion = MemberOrDefault(*obj, "version", 0); + statedVersion = MemberOrDefault(*obj, "version", "0.0"); if (Value* profile = FindObject(*obj, "profile")) { ReadMember(*profile, "api", this->profile.api); @@ -1243,14 +1076,16 @@ inline void AssetMetadata::Read(Document& doc) } } - version = std::max(statedVersion, version); + float statedFloatVersion = std::strtof(statedVersion.c_str(), 0); + + version = std::max(statedFloatVersion, version); if (version == 0) { - // if missing version, we'll assume version 1.0... - version = 1; + // if missing version, we'll assume version 2.0... + version = 2; } - if (version != 1) { + if (version != 2) { char msg[128]; ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.1f", version); throw DeadlyImportError(msg); @@ -1361,12 +1196,14 @@ inline void Asset::Load(const std::string& pFile, bool isBinary) mDicts[i]->AttachToDocument(doc); } - - // Read the "scene" property, which specifies which scene to load // and recursively load everything referenced by it - if (Value* scene = FindString(doc, "scene")) { - this->scene = scenes.Get(scene->GetString()); + if (Value* scene = FindUInt(doc, "scene")) { + unsigned int sceneIndex = scene->GetUint(); + + Ref s = scenes.Get(sceneIndex); + + this->scene = s; } // Clean up From 47c7c3cf506166b483fd4e44e20a58cd1107e763 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 25 Aug 2017 16:09:07 -0400 Subject: [PATCH 090/305] Disambiguate Get methods --- code/glTF2Asset.h | 4 +-- code/glTF2Asset.inl | 59 +++++++++++++++++++++++++-------------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 2e7a694d8..2ec2b84eb 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -1046,9 +1046,9 @@ namespace glTF2 LazyDict(Asset& asset, const char* dictId, const char* extId = 0); ~LazyDict(); - Ref Get(const char* id); + Ref Retrieve(unsigned int i); + Ref Get(unsigned int i); - Ref Get(const std::string& pID) { return Get(pID.c_str()); } Ref Create(const char* id); Ref Create(const std::string& id) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 268deb8a8..97e507ee6 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -194,6 +194,14 @@ inline void LazyDict::DetachFromDocument() template Ref LazyDict::Get(unsigned int i) +{ + + return Ref(mObjs, i); + +} + +template +Ref LazyDict::Retrieve(unsigned int i) { std::string id = std::string(mDictId) + "[" + std::to_string(i) + "]"; @@ -442,7 +450,7 @@ inline void BufferView::Read(Value& obj, Asset& r) { if (Value* bufferVal = FindUInt(obj, "buffer")) { - buffer = r.buffers.Get(bufferVal->GetUint()); + buffer = r.buffers.Retrieve(bufferVal->GetUint()); } byteOffset = MemberOrDefault(obj, "byteOffset", 0u); @@ -457,7 +465,7 @@ inline void Accessor::Read(Value& obj, Asset& r) { if (Value* bufferViewVal = FindUInt(obj, "bufferView")) { - bufferView = r.bufferViews.Get(bufferViewVal->GetUint()); + bufferView = r.bufferViews.Retrieve(bufferViewVal->GetUint()); } byteOffset = MemberOrDefault(obj, "byteOffset", 0u); @@ -615,7 +623,7 @@ inline void Image::Read(Value& obj, Asset& r) ReadMember(*ext, "mimeType", mimeType); if (Value* bufferViewVal = FindUInt(*ext, "bufferView")) { - Ref bv = r.bufferViews.Get(bufferViewVal->GetUint()); + Ref bv = r.bufferViews.Retrieve(bufferViewVal->GetUint()); if (bv) { mDataLength = bv->byteLength; mData = new uint8_t[mDataLength]; @@ -690,11 +698,11 @@ inline void Sampler::SetDefaults() inline void Texture::Read(Value& obj, Asset& r) { if (Value* sourceVal = FindUInt(obj, "source")) { - source = r.images.Get(sourceVal->GetUint()); + source = r.images.Retrieve(sourceVal->GetUint()); } if (Value* samplerVal = FindUInt(obj, "sampler")) { - sampler = r.samplers.Get(samplerVal->GetUint()); + sampler = r.samplers.Retrieve(samplerVal->GetUint()); } } @@ -704,7 +712,7 @@ namespace { //@TODO: update this format if (Value* prop = FindMember(vals, propName)) { if (prop->IsUint()) { - out.texture = r.textures.Get(prop->GetUint()); + out.texture = r.textures.Retrieve(prop->GetUint()); } else { ReadValue(*prop, out.color); @@ -820,7 +828,7 @@ inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) if (Value* attrs = FindObject(primitive, "attributes")) { for (Value::MemberIterator it = attrs->MemberBegin(); it != attrs->MemberEnd(); ++it) { - if (!it->value.IsString()) continue; + if (!it->value.IsUint()) continue; const char* attr = it->name.GetString(); // Valid attribute semantics include POSITION, NORMAL, TEXCOORD, COLOR, JOINT, JOINTMATRIX, // and WEIGHT.Attribute semantics can be of the form[semantic]_[set_index], e.g., TEXCOORD_0, TEXCOORD_1, etc. @@ -831,17 +839,17 @@ inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) if (GetAttribVector(prim, attr, vec, undPos)) { size_t idx = (attr[undPos] == '_') ? atoi(attr + undPos + 1) : 0; if ((*vec).size() <= idx) (*vec).resize(idx + 1); - (*vec)[idx] = pAsset_Root.accessors.Get(it->value.GetUint()); + (*vec)[idx] = pAsset_Root.accessors.Retrieve(it->value.GetUint()); } } } if (Value* indices = FindUInt(primitive, "indices")) { - prim.indices = pAsset_Root.accessors.Get(indices->GetUint()); + prim.indices = pAsset_Root.accessors.Retrieve(indices->GetUint()); } if (Value* material = FindUInt(primitive, "material")) { - prim.material = pAsset_Root.materials.Get(material->GetUint()); + prim.material = pAsset_Root.materials.Retrieve(material->GetUint()); } } } @@ -992,9 +1000,9 @@ inline void Node::Read(Value& obj, Asset& r) this->children.reserve(children->Size()); for (unsigned int i = 0; i < children->Size(); ++i) { Value& child = (*children)[i]; - if (child.IsNumber()) { + if (child.IsUint()) { // get/create the child node - Ref chn = r.nodes.Get(child.GetUint()); + Ref chn = r.nodes.Retrieve(child.GetUint()); if (chn) this->children.push_back(chn); } } @@ -1010,22 +1018,21 @@ inline void Node::Read(Value& obj, Asset& r) ReadMember(obj, "rotation", rotation); } - if (Value* meshes = FindArray(obj, "meshes")) { - unsigned numMeshes = (unsigned)meshes->Size(); + if (Value* mesh = FindUInt(obj, "mesh")) { + //unsigned numMeshes = (unsigned)meshes->Size(); + unsigned numMeshes = 1; - std::vector meshList; + //std::vector meshList; this->meshes.reserve(numMeshes); - for (unsigned i = 0; i < numMeshes; ++i) { - if ((*meshes)[i].IsNumber()) { - Ref mesh = r.meshes.Get((*meshes)[i].GetUint()); - if (mesh) this->meshes.push_back(mesh); - } - } + + Ref meshRef = r.meshes.Retrieve((*mesh).GetUint()); + + if (meshRef) this->meshes.push_back(meshRef); } if (Value* camera = FindUInt(obj, "camera")) { - this->camera = r.cameras.Get(camera->GetUint()); + this->camera = r.cameras.Retrieve(camera->GetUint()); if (this->camera) this->camera->id = this->id; } @@ -1037,7 +1044,7 @@ inline void Node::Read(Value& obj, Asset& r) if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { if (Value* light = FindUInt(*ext, "light")) { - this->light = r.lights.Get(light->GetUint()); + this->light = r.lights.Retrieve(light->GetUint()); } } @@ -1049,8 +1056,8 @@ inline void Scene::Read(Value& obj, Asset& r) { if (Value* array = FindArray(obj, "nodes")) { for (unsigned int i = 0; i < array->Size(); ++i) { - if (!(*array)[i].IsNumber()) continue; - Ref node = r.nodes.Get((*array)[i].GetUint()); + if (!(*array)[i].IsUint()) continue; + Ref node = r.nodes.Retrieve((*array)[i].GetUint()); if (node) this->nodes.push_back(node); } @@ -1201,7 +1208,7 @@ inline void Asset::Load(const std::string& pFile, bool isBinary) if (Value* scene = FindUInt(doc, "scene")) { unsigned int sceneIndex = scene->GetUint(); - Ref s = scenes.Get(sceneIndex); + Ref s = scenes.Retrieve(sceneIndex); this->scene = s; } From 4d59dee5ea25b7f1a7512348de6bc80a09a6d3fa Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 25 Aug 2017 16:52:44 -0400 Subject: [PATCH 091/305] Cache retrieved items via an original index map --- code/glTF2Asset.h | 15 ++++++++------- code/glTF2Asset.inl | 11 +++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 2ec2b84eb..2b37d6a0e 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -382,6 +382,7 @@ namespace glTF2 struct Object { int index; //!< The index of this object within its property container + int oIndex; //!< The original index of this object defined in the JSON std::string id; //!< The globally unique ID used to reference this object std::string name; //!< The user-defined name of this object @@ -1025,14 +1026,14 @@ namespace glTF2 friend class Asset; friend class AssetWriter; - typedef typename std::gltf_unordered_map< std::string, unsigned int > Dict; + typedef typename std::gltf_unordered_map< unsigned int, unsigned int > Dict; - std::vector mObjs; //! The read objects - Dict mObjsById; //! The read objects accessible by id - const char* mDictId; //! ID of the dictionary object - const char* mExtId; //! ID of the extension defining the dictionary - Value* mDict; //! JSON dictionary object - Asset& mAsset; //! The asset instance + std::vector mObjs; //! The read objects + Dict mObjsByOIndex; //! The read objects accessible by original index + const char* mDictId; //! ID of the dictionary object + const char* mExtId; //! ID of the extension defining the dictionary + Value* mDict; //! JSON dictionary object + Asset& mAsset; //! The asset instance void AttachToDocument(Document& doc); void DetachFromDocument(); diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 97e507ee6..062d8318d 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -204,10 +204,8 @@ template Ref LazyDict::Retrieve(unsigned int i) { - std::string id = std::string(mDictId) + "[" + std::to_string(i) + "]"; - - typename Dict::iterator it = mObjsById.find(id); - if (it != mObjsById.end()) { // already created? + typename Dict::iterator it = mObjsByOIndex.find(i); + if (it != mObjsByOIndex.end()) {// already created? return Ref(mObjs, it->second); } @@ -227,7 +225,8 @@ Ref LazyDict::Retrieve(unsigned int i) } T* inst = new T(); - inst->id = id; + inst->id = std::string(mDictId) + "[" + std::to_string(i) + "]"; + inst->oIndex = i; ReadMember(obj, "name", inst->name); inst->Read(obj, mAsset); @@ -239,7 +238,7 @@ Ref LazyDict::Add(T* obj) { unsigned int idx = unsigned(mObjs.size()); mObjs.push_back(obj); - mObjsById[obj->id] = idx; + mObjsByOIndex[obj->oIndex] = idx; mAsset.mUsedIds[obj->id] = true; return Ref(mObjs, idx); } From 39172feb3e3e9bb0c185209ed996867414ff0f44 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 25 Aug 2017 17:58:41 -0400 Subject: [PATCH 092/305] Start reading pbr materials --- code/glTF2Asset.inl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 062d8318d..27fdd1c01 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -733,6 +733,12 @@ inline void Material::Read(Value& material, Asset& r) ReadMember(*values, "shininess", shininess); } + if (Value* values = FindObject(material, "pbrMetallicRoughness")) { + ReadMaterialProperty(r, *values, "baseColorFactor", this->diffuse); + } + + ReadMember(material, "doubleSided", doubleSided); + if (Value* extensions = FindObject(material, "extensions")) { if (r.extensionsUsed.KHR_materials_common) { if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { From 67eb3b06087021490e43d6c74e05dcd88c6d921a Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 27 Aug 2017 23:47:31 -0400 Subject: [PATCH 093/305] temporarily disable gltf exporting of animations and skins --- code/glTF2Exporter.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 7daef7000..68f2f4420 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -128,7 +128,7 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai ExportScene(); - ExportAnimations(); + //ExportAnimations(); AssetWriter writer(*mAsset); @@ -414,7 +414,7 @@ Ref FindSkeletonRootJoint(Ref& skinRef) return parentNodeRef; } -void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref& bufferRef, Ref& skinRef, std::vector& inverseBindMatricesData) +/*void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref& bufferRef, Ref& skinRef, std::vector& inverseBindMatricesData) { if (aimesh->mNumBones < 1) { return; @@ -491,7 +491,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, RefHasBones()) { - ExportSkin(*mAsset, aim, m, b, skinRef, inverseBindMatricesData); - } + /*************** Skins ****************/ + /*if(aim->HasBones()) { + ExportSkin(*mAsset, aim, m, b, skinRef, inverseBindMatricesData); + }*/ /****************** Compression ******************/ ///TODO: animation: weights, joints. @@ -942,7 +942,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref bufferRef = mAsset->buffers.Get(unsigned (0)); @@ -963,7 +963,7 @@ void glTF2Exporter::ExportAnimations() name = mAsset->FindUniqueID(name, "animation"); Ref animRef = mAsset->animations.Create(name); - /******************* Parameters ********************/ + // Parameters ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); for (unsigned int j = 0; j < 3; ++j) { @@ -1011,7 +1011,7 @@ void glTF2Exporter::ExportAnimations() // } } // End: for-loop mNumAnimations -} +} */ #endif // ASSIMP_BUILD_NO_GLTF_EXPORTER From b42d785afebd313f8a531dc8cf832d80ecd50910 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 27 Aug 2017 23:47:54 -0400 Subject: [PATCH 094/305] Start managing and importing gltf2 pbr materials --- code/glTF2Asset.h | 40 +++++++++++++++++++++++++++++----------- code/glTF2Asset.inl | 19 +++++++++++++++---- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 2b37d6a0e..5d99735b2 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -671,11 +671,18 @@ namespace glTF2 inline void SetData(uint8_t* data, size_t length, Asset& r); }; - //! Holds a material property that can be a texture or a color - struct TexProperty + struct ColorProperty + { + union { + vec4, vec3 + } color; + } + + //! Holds a material property that can be a texture or a color (fallback for glTF 1) + struct FallbackTexProperty { Ref texture; - vec4 color; + ColorProperty color; }; //! The material appearance of a primitive. @@ -694,16 +701,27 @@ namespace glTF2 Technique_CONSTANT }; - TexProperty ambient; - TexProperty diffuse; - TexProperty specular; - TexProperty emission; - Ref normal; + //PBR metallic roughness properties + ColorProperty baseColor; + Ref baseColorTexture; + Ref metallicRoughnessTexture; + float metallicFactor; + float roughnessFactor; + //other basic material properties + Ref normalTexture; + Ref occlusionTexture; + Ref emissiveTexture; + ColorProperty emissiveFactor; + std::string alphaMode; + float alphaCutoff; bool doubleSided; - bool transparent; - float transparency; - float shininess; + + //fallback material properties (compatible with non-pbr defintions) + FallbackTexProperty diffuse; + FallbackTexProperty emissive; + FallbackTexProperty specular; + Ref normal; Technique technique; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 27fdd1c01..bf8cb5a36 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -725,18 +725,29 @@ inline void Material::Read(Value& material, Asset& r) SetDefaults(); if (Value* values = FindObject(material, "values")) { - ReadMaterialProperty(r, *values, "ambient", this->ambient); - ReadMaterialProperty(r, *values, "diffuse", this->diffuse); - ReadMaterialProperty(r, *values, "specular", this->specular); + ReadMember(*values, "transparency", transparency); - ReadMember(*values, "shininess", shininess); } if (Value* values = FindObject(material, "pbrMetallicRoughness")) { + //pbr + ReadMaterialProperty(r, *values, "baseColorFactor", this->baseColor); + ReadMaterialProperty(r, *values, "baseColorTexture", this->baseColorTexture); + + //non-pbr fallback ReadMaterialProperty(r, *values, "baseColorFactor", this->diffuse); + ReadMaterialProperty(r, *values, "baseColorTexture", this->diffuse); + + ReadMember(*values, "metallicFactor", metallicFactor); } + ReadMaterialProperty(r, *values, "normalTexture", this->normalTexture); + ReadMaterialProperty(r, *values, "normalTexture", this->normal); + ReadMaterialProperty(r, *values, "occlusionTexture", this->occlusionTexture); + ReadMaterialProperty(r, *values, "emissiveTexture", this->emissiveTexture); + ReadMember(*values, "metallicFactor", emissiveFactor); + ReadMember(material, "doubleSided", doubleSided); if (Value* extensions = FindObject(material, "extensions")) { From 6b4286abf69a6bdff733dcf88a13891c21892985 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 27 Aug 2017 23:50:22 -0400 Subject: [PATCH 095/305] check in gltf2 models to test directory Remove un-needed test models --- .../BoxTextured-glTF-Binary/BoxTextured.glb | Bin 0 -> 4696 bytes .../BoxTextured.gltf | 181 +++++++++++ .../BoxTextured.gltf | 197 ++++++++++++ .../BoxTextured0.bin | Bin 0 -> 840 bytes .../CesiumLogoFlat.png | Bin 0 -> 2433 bytes .../BoxTextured.gltf | 282 ++++++++++++++++++ .../BoxTextured0.bin | Bin 0 -> 840 bytes .../BoxTextured0.vert | 17 ++ .../BoxTextured1.frag | 29 ++ .../CesiumLogoFlat.png | Bin 0 -> 2433 bytes .../glTF2/BoxTextured-glTF/BoxTextured.gltf | 181 +++++++++++ .../glTF2/BoxTextured-glTF/BoxTextured0.bin | Bin 0 -> 840 bytes .../glTF2/BoxTextured-glTF/CesiumLogoFlat.png | Bin 0 -> 2433 bytes 13 files changed, 887 insertions(+) create mode 100644 test/models/glTF2/BoxTextured-glTF-Binary/BoxTextured.glb create mode 100644 test/models/glTF2/BoxTextured-glTF-Embedded/BoxTextured.gltf create mode 100644 test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured.gltf create mode 100644 test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured0.bin create mode 100644 test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/CesiumLogoFlat.png create mode 100644 test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured.gltf create mode 100644 test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured0.bin create mode 100644 test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured0.vert create mode 100644 test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured1.frag create mode 100644 test/models/glTF2/BoxTextured-glTF-techniqueWebGL/CesiumLogoFlat.png create mode 100644 test/models/glTF2/BoxTextured-glTF/BoxTextured.gltf create mode 100644 test/models/glTF2/BoxTextured-glTF/BoxTextured0.bin create mode 100644 test/models/glTF2/BoxTextured-glTF/CesiumLogoFlat.png diff --git a/test/models/glTF2/BoxTextured-glTF-Binary/BoxTextured.glb b/test/models/glTF2/BoxTextured-glTF-Binary/BoxTextured.glb new file mode 100644 index 0000000000000000000000000000000000000000..03cd2acf941a0838722aa3e7243d33a8f459c22e GIT binary patch literal 4696 zcmbUj2{@GN`cEYsB}<5iNh)L+J7qbxXqXs8A%mG|WXu>dvXy-)rLt?GvUDV*bV!TN z*s_;#$i63}>?+&+|1r~XyXT(nbHC?#|9AV|{r%<{oQJhB8vp=yd;mb&3;<>*q(uM> z?d^@Bz_bEjIE*KTjHVFDFfEt?5`oY+)Q6iOtc_uEFdq!r8&C8E6>xb)*f}|vw<~xV zrlrWFykS}=17My+Hw;86I-YZs`_2WxTumj(q?Nfo0;Z)T z2RnvDnOd77Enr%3IheJPodFVQWq3*vrlkxPiT8BFyFw;lnFO#ln3f77dkh(m_F%^J zL=!+c%pA%B$^q>PT6h!5YaMZ+VzC&qEgo~0;l#y{fUk z)qbd#xn2Jj4bB)UxUr0xuML&tLN>=x&>kLmS1TeFciIyiIJ8zSXm5-G(St~~#`sdG zWN@MZ&2`ULpxTHYx#4*ZX`dV`gnj7Oiz`9^sn(A;(#?ZeDG&R6U zoh745DDagis%WaOan?)@%JQ$EYyQ^w|5$T%HEUo-R^m45IA{|8^+jai8(X)+gwU>m>#!d75AzzgsJ{2=fG0>BPH5D)@k2XNt-g~?Vf z5iT&$R?{PfRv@NBz_Ahh%C)zSgQ)Nr!rG{-t7~a_d1ZCgA!^_1>T2Rc?=Ops=$M1; zoo(b~!@xAl+4*UwXqm^AY3aowF*(@0KN2VBm%=k0?!CB#ja6yxEE2z(-V4IL+!xqo8h_A_Q|rwyqdP+k%`{E53RKB>CWM%;o)JAyxxcP z{XxZpN#(SMFW(={-sou7+k4tOyJE|RZ&iR**9=&KVQl=sxf`>}2S|T<;#ZhL^ct$!PlY`16PdQf zZNFpkCW$JnuM77#E>GiKAUzng%NekUiygiaODc9wvVC&b@jf3q;cBZ~n6p%BnP7yH zNOBE*yHeT=f0>P7y}nQDwddt3EAQSPxjZywT7{O!88Qp#ns}YYy+1q)qagSUHsH)@T0NmLHB1;CE6?2 zk{|fYh|QLFwu;>{=RW6tDj~`khqMZ{$Z?&9QA%~rN8N6B+b8TlrIVhcbUe9>TH{-!*Nu!bTe3}+>czz_ymcmX`E(5&8xkndIUn!8NAs|T9oPKt zFE~SYX$W&+j|WQE1lBst&b^7xWJj3uD0VI)R6SVvd@ab^gfu*u-)0`iQx6ecBm5^)w@56T^J2gSUvw_qUwY9wccdk zd(&7l{BWL?MnHQvUH7=DtOLDH??StDd6eAP5l>l?*=VZOm}tZ=1yk6=#Z4p=!v0jv z>O@$F+G&}tdnhV;;pk1xl=n;QCtR_E@_BG9SRGVZmG%By@~QaLKc0Ur@+h4mXl;VaA22% z)HbheBT1X)LQs{li;ureCZCF6Oa2;uc3jU%;K7kYsQJ>eqLfZ|nx25sg&S^)LK&vk z9UaXggxfYAydtBkL3XD$jeDqq+0KbPDvx{&7m9UCGyrTghE3v$d3ENnngIbtKQV`o zMTy~C1_ex}H4p!3=`MPV3BWJnfIuvxPh8LvkE-dEqEt~e_rKBzpQC*?$L=IM3X-WdKM;SnO2 zOK|7s+P(etq1izhGAM!a>>-OdQk90x-t@DVd_^UHIcsGmPghR4s{J(&p_+g7eG$@s zCK2xvM8)`1wKO6gBM{kHEoaL7nx4iR`&Kt=PnC))*f3ZHaU{ z(Z`-ePkl(`z;W5uqOz;{*mtIPJJMEtiwDKCLL|8Q=#2vTln|@nyIW+s_MQ$l3`piL zTNP5SGK4*z5ZyJopzdd0{=8edGZcQutfuCaG{GpjXIH;oN^ai7m(3loBPjK^S~FEv zNC8<%4RaU~axqWlN3s6mf)?=v)A4v8lFE+jp+fonqk?G*H;W2lNr=x@CdkyHW7p;f zII6L?tycVYUz;D}tiGSNOwU^$Cej_9-Gp@)EBf!2FDKGwoD_slpen?p?nQ7F2b6Lj zjjf{Rxz)c5n=miFEZOBb?~&VL+7M7 zZLO@mF(qYIaX0(QF;<0@dyK^Lw=Eo((Y&28*PP_xyRXo1H|K)(ahY!o=VvTeyn1Ba zy?PX1B0c(7%(6z`H&yvMaHX8J-j`RrG&RxoPE03Dq6>8Z^Qb*H_r6~>^g~*(UXoxdlNdhK+VOk=Cp!ifNI#CMeS-$)D?AWqi`z=7{xu+F_b4KrM}^ z7w6cPzHOm-r0rd_SjLt}g@!NLw7Y^~J8tCV81&oi4jd21={j6xi!2!G%;hQ>nrrtk zyyY(wSi7JIsL!?Fo|%kMG)F5_KJWQ_Vt-uR1T7f&S|>jez?o}!qGiV^*CTq8xXS&L z%i;P<1!j@Twr}3MIqbd2=k_cD(Q8AzF+87Yed!JP^W>B2PYY>! z+k=nF*n95jlyH6rI88bw+;sjFR3vB5&b|~@CR%jgk&u?Tabc$sW z+s03|SsX(46J1hbhW&i1HpC+DlA`;)KkST>o2%S}|019TJlQ<#W!HpD*%Z}q{YoDx zbcA0;!+^UEhr9XcPt^8jNm50rG;31H zropwtw2AMsvOGuRaG|JA;%s=iQ4GfZK{Y0Ss5Up_?a4pH*po7Zx&|idv=MDN-;CJ; z562XbkL5(&dR&j-{Pg>2tg1vP)kUX6^5PrCWW15wML(4dQxr9nc14j$wwjiq^!>CK zodkY*;l?y-p|M8BMiOa=Q!$GbcIIX(v! zHAaLMISNdh+Xub=!Y7KT`fcg8nTz7yjO9B1DjFBR!7yL%!9ZkqJM~qSWTjW*??(;n zF0}5dolo(}ylTp88`0suSFpxDJV#A6e0Xm0@cWzIjgCQoywMW6Bq#>{rvglkEDZC1 Jbq)o|e*xHM<_Q1* literal 0 HcmV?d00001 diff --git a/test/models/glTF2/BoxTextured-glTF-Embedded/BoxTextured.gltf b/test/models/glTF2/BoxTextured-glTF-Embedded/BoxTextured.gltf new file mode 100644 index 000000000..c9c461495 --- /dev/null +++ b/test/models/glTF2/BoxTextured-glTF-Embedded/BoxTextured.gltf @@ -0,0 +1,181 @@ +{ + "asset": { + "generator": "COLLADA2GLTF", + "version": "2.0" + }, + "scene": 0, + "scenes": [ + { + "nodes": [ + 0 + ] + } + ], + "nodes": [ + { + "children": [ + 1 + ], + "matrix": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + }, + { + "mesh": 0 + } + ], + "meshes": [ + { + "primitives": [ + { + "attributes": { + "NORMAL": 1, + "POSITION": 2, + "TEXCOORD_0": 3 + }, + "indices": 0, + "mode": 4, + "material": 0 + } + ], + "name": "Mesh" + } + ], + "accessors": [ + { + "bufferView": 0, + "byteOffset": 0, + "componentType": 5123, + "count": 36, + "max": [ + 23 + ], + "min": [ + 0 + ], + "type": "SCALAR" + }, + { + "bufferView": 1, + "byteOffset": 0, + "componentType": 5126, + "count": 24, + "max": [ + 1.0, + 1.0, + 1.0 + ], + "min": [ + -1.0, + -1.0, + -1.0 + ], + "type": "VEC3" + }, + { + "bufferView": 1, + "byteOffset": 288, + "componentType": 5126, + "count": 24, + "max": [ + 0.5, + 0.5, + 0.5 + ], + "min": [ + -0.5, + -0.5, + -0.5 + ], + "type": "VEC3" + }, + { + "bufferView": 2, + "byteOffset": 0, + "componentType": 5126, + "count": 24, + "max": [ + 6.0, + 1.0 + ], + "min": [ + 0.0, + 0.0 + ], + "type": "VEC2" + } + ], + "materials": [ + { + "pbrMetallicRoughness": { + "baseColorTexture": { + "index": 0 + }, + "metallicFactor": 0.0 + }, + "name": "Texture" + } + ], + "textures": [ + { + "sampler": 0, + "source": 0 + } + ], + "images": [ + { + "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAANMAAADTCAMAAAAs2dbrAAAAsVBMVEXc3Nz8/f3+//9chyf///+VsHP4+/thiy3Z29dyl0R8nlH09vJeiCqzxZ6ft4GLqWatwZXw9vyEpFynvYxmjjTS3cNrkjvu8urn8vre7fjY6ffL08K6yqbEz7XT183K4vTQ5fW62fC/zq3J17fn7d/g6NbC3fLb5c/l5eVsrd+wzOF/t+OWw+WGu+TE099vr+CmyON5tOJ0seG50OHL1t7S2d7Z29yOv+Wdx+emzuyx1O6AHuoWAAAIi0lEQVR4XuzUS3LEIAxF0WznIYk/xun9LyxDRqnqGANq4rODW4L39d88Ho/H43HkZPzJVYgoxhiIpDJ7k2z5wJpizSkBv4vCJn1OWTZMeE9gY/Xfx3DE30Q2WfGBvOAa8lZnEKFH0JZ1GEE/MnpGw54R93Cs41ip4k6SjuWPjnC3YJZWtaJdqhJhlJDWLINgJJm/FoUxGpfJ0xAxnvMTv1UWzEF21pG8wzT+2OZIjeTxScZhLmdGz13FfHXoAGbCCmHgVCSHNVwaleSxjh8z4Yxmj09VBGtRGbAOq1G+OSlgvZC3SwLijVE2Qodot7lS4+xGSS1quyS4O/5UIegSSm/SUaENHZ1NDH1qX5SHRmdPUoJOqWPyInS6Pn6HoNlkJxh68T6fqTFXkkrERPL9en0L3ueufKkf6uxmt1EYiAN4sP7i0pNPABeItEIdeV0vJMTA+z/YaqWo3WSS1gn2lP4ewFHEfNqexLi5hAIAvTgKZZ+4myQxpsSZUtomiD75yPM5Pqg8uDK5cbORZzT+p3JDgfxma16LK3WaceLUkxQPJjj6hnGbo2sHpkkxzB4didFgcgp23OJQ5BS4Pn6TeiM5PThYCva2vQmCHDgYCmZXfCbJfMpc9PXQk6QGTE0PMNv7TDSBmaM/IXqSVeKKdsSsy6gjCfMZLijWSVaXvomkzbiwRF/jR0fS3JThXTbTw8aAVVCcbXFW2wRv8oa+hVmaumoWS88YTgKFXNpBYHrd1GXfaaCfyI0CK7u0g0DoSfMCoSduTBF6Zto3bV033ewdiel/L/9+te3G6HOR35d4p/JmcjI9rVI4e9nd0z9XSStc08tAidkmw4cs6khuK3BK6ZlS6huFC6/xZj23z3BHbSiZOceVIto2aCrcp1N1BteByXe3DfQgq/EZNVMKQwUui5ROPscXFoqvL3HLa5R0sjkg/6eGEjcVMQajXuNraqK4XA3ufoey9AhXIURmUzyDhCbUya0/nFN6SHIbyK0vERMCqZZW4ikcXCQOcQ7nlgT1gVPFjvuz+vCXX4UCl/mEd+tAXhSsSDxT9rq/tJjbasJAFEWJm2QwF6NJVNra0hptoFVsERrp/39YXwTrbDWZOcf1GrLDhrMyF45elUMAVTjlR5NvrctNIsniFDB5HVzstJWEB+MUR/I5PdVRahtQcF3hSNxEkoMGhw+SAidmU1bqLjIFI5wwS670IwhPYvzHNCRweweZBiHOeBH8yik8KnBOtWal9GUqcc4T38D6h49gM0zoNLXTlqmGBQ/fXhDOhKyUskyrChYZ78ol4cyCpn+jKlMUw6bkTpJwxjgqJZ93jHgbIQt3Ukou0zOYkDv5rUw1rpDmtE9685VpYn91ZXp1OjiFs0xMpqXUa795z7mTRnjH0jvxuh77pXkP0a/TRh7erdSDh1ItybTERQruJA5nNJTia4+16dvp4C4Th+srxfOeDKHVaccyUTiRNnan6FNNpm6fvhTCmVlil3JTqh3wvDt1ksvE5PTeu5ZM3evTh0o4MxZcY+4eO2QSdeobzpgpK6UjU/d+b68TzszmvqvUhmRa4Aal05mwdQhnClbKU6bG4AYZn91dwv94t+MepYEgCuDTzhSudIFSgKIHKtwlaIBEe5cTv/8XM8ZEIrPtGxzw9zd5TcO83SbbgjKhSsnJXyZtRZfeHOFAmKut7fgF2siFdMyd1qQ0/nB7pZbN9WUacrcPpOwd4bev1B6USeuTdnA0FZqCF67h6Rae95y0r45wTO9SR0+ZtFCR9uIIx2agUv55H5H26gnHqp7906395W/TXWBkGD/7xOGiw/2VwvNeLxhaU8Qeh6eFCrd7ugxLjsbTrWTM2IPtTbeNIdxu9myr1EnNe8lYThFvL65wLAyilcLzPg+MZRTz6gzHRoaTgWYZKxNWUlRzZbi/UpKc4MNlUrHFI0Udrg73Vyo5oKP10pZseCexeR8J9ztXKvrNfvNR/nHeK2px3jC+T2zh/kql6ebPxnhcCph3VCett/zx7dfYHT6lINxTKWWy2aurqnnHdYqrRdLJu2UiAsId+s8S0XJVmbJNTm0+iwLCPZXC0jkbZdQqkRYq3CETq3rGRltqVUhUqsI9wlYAdbqFzB4IDp8O99Ln19iIHaMHhi/N+KbCohaDFZutqUPhWB9864RnP1yQAoav5NurCrjk9dlsRJ16KnzK9zAAN/XUZ7tH6nQ56cWY7yPvWiiSku3CgAD5yy7newmrRFrUFV9jSEAhZ8Uo8P2EQfyvSoYzVtDmZFwlihJku4VM31Vvm/N1pgQVv6PnWZ//g2pby1m6my5+cls3ubHCQBCASypZbiPbIGDYAF7wXpScYO5/tCxZBgb8g78btKpdbZ6kA/708/Vah35kKrbxy+v7/9frX9dOPE/hb2KYi+V5dsYBjk+icIjhc+gZhyz1xQRp+BQ6ALUF5XFYz2eYBIfNmo/gcILnEzQ4QwzLZ2ecstVTELuWpTOCk8LEwq04batm83Zvz5I1gg+IeczftY7L2+FDHUul8DFVzWPaSc8SjTMuCIblsRt2lfSEw0WLZVmsx2Wu5Mqro9FbwR0GlqMX3OLtyztM16n6RgJ8PYu36+qph52zzEwJ7rZoZjUggnViPtohitAwl2lFLIp5NAHxdDZTO8S0GqY2OkQmqp69220T07EDkggtU2lmpOLSRKUHJCTeMro2IK25Z1xmRXqLYTxjJ8jh7QzjGAdBLhJlKu0DcpLO8F7TEJDd1lrexjhBCd7Bj7yDVivKIe56WL/t2jEKwCAMheEsYirEBgNdohl6CO9/tM6FgrRdFPJNWX/I+BIbTMao/MhKYjAlo/3LE8bCChND5RLfLXxPhNl1wMYhxXHOtktDWInWHMr2mHakkEkRVoXaaiUWyVmEqTY1hOX1+z3mnHPOXX/B0DoXjBYdAAAAAElFTkSuQmCC" + } + ], + "samplers": [ + { + "magFilter": 9729, + "minFilter": 9986, + "wrapS": 10497, + "wrapT": 10497 + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": 768, + "byteLength": 72, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 0, + "byteLength": 576, + "byteStride": 12, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 576, + "byteLength": 192, + "byteStride": 8, + "target": 34962 + } + ], + "buffers": [ + { + "byteLength": 840, + "uri": "data:application/octet-stream;base64,AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAPwAAAL8AAAA/AAAAPwAAAD8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAD8AAAC/AAAAPwAAAD8AAAC/AAAAPwAAAL8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAC/AAAAvwAAAL8AAAC/AAAAvwAAAL8AAAA/AAAAvwAAAD8AAAA/AAAAvwAAAL8AAAC/AAAAvwAAAD8AAAC/AAAAvwAAAL8AAAC/AAAAvwAAAD8AAAC/AAAAPwAAAL8AAAC/AAAAPwAAAD8AAAC/AADAQAAAAAAAAKBAAAAAAAAAwED+/38/AACgQP7/fz8AAIBAAAAAAAAAoEAAAAAAAACAQAAAgD8AAKBAAACAPwAAAEAAAAAAAACAPwAAAAAAAABAAACAPwAAgD8AAIA/AABAQAAAAAAAAIBAAAAAAAAAQEAAAIA/AACAQAAAgD8AAEBAAAAAAAAAAEAAAAAAAABAQAAAgD8AAABAAACAPwAAAAAAAAAAAAAAAP7/fz8AAIA/AAAAAAAAgD/+/38/AAABAAIAAwACAAEABAAFAAYABwAGAAUACAAJAAoACwAKAAkADAANAA4ADwAOAA0AEAARABIAEwASABEAFAAVABYAFwAWABUA" + } + ] +} diff --git a/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured.gltf b/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured.gltf new file mode 100644 index 000000000..b286239fa --- /dev/null +++ b/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured.gltf @@ -0,0 +1,197 @@ +{ + "asset": { + "generator": "COLLADA2GLTF", + "version": "2.0" + }, + "scene": 0, + "scenes": [ + { + "nodes": [ + 0 + ] + } + ], + "nodes": [ + { + "children": [ + 1 + ], + "matrix": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ] + }, + { + "mesh": 0 + } + ], + "meshes": [ + { + "primitives": [ + { + "attributes": { + "NORMAL": 1, + "POSITION": 2, + "TEXCOORD_0": 3 + }, + "indices": 0, + "mode": 4, + "material": 0 + } + ], + "name": "Mesh" + } + ], + "accessors": [ + { + "bufferView": 0, + "byteOffset": 0, + "componentType": 5123, + "count": 36, + "max": [ + 23 + ], + "min": [ + 0 + ], + "type": "SCALAR" + }, + { + "bufferView": 1, + "byteOffset": 0, + "componentType": 5126, + "count": 24, + "max": [ + 1.0, + 1.0, + 1.0 + ], + "min": [ + -1.0, + -1.0, + -1.0 + ], + "type": "VEC3" + }, + { + "bufferView": 1, + "byteOffset": 288, + "componentType": 5126, + "count": 24, + "max": [ + 0.5, + 0.5, + 0.5 + ], + "min": [ + -0.5, + -0.5, + -0.5 + ], + "type": "VEC3" + }, + { + "bufferView": 2, + "byteOffset": 0, + "componentType": 5126, + "count": 24, + "max": [ + 6.0, + 1.0 + ], + "min": [ + 0.0, + 0.0 + ], + "type": "VEC2" + } + ], + "materials": [ + { + "pbrMetallicRoughness": { + "baseColorTexture": { + "index": 0 + }, + "metallicFactor": 0.0 + }, + "extensions": { + "KHR_materials_pbrSpecularGlossiness": { + "diffuseTexture": { + "index": 0 + }, + "specularFactor": [ + 0.20000000298023225, + 0.20000000298023225, + 0.20000000298023225 + ], + "glossinessFactor": 1.0 + } + }, + "name": "Texture" + } + ], + "textures": [ + { + "sampler": 0, + "source": 0 + } + ], + "images": [ + { + "uri": "CesiumLogoFlat.png" + } + ], + "samplers": [ + { + "magFilter": 9729, + "minFilter": 9986, + "wrapS": 10497, + "wrapT": 10497 + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": 768, + "byteLength": 72, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 0, + "byteLength": 576, + "byteStride": 12, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 576, + "byteLength": 192, + "byteStride": 8, + "target": 34962 + } + ], + "buffers": [ + { + "byteLength": 840, + "uri": "BoxTextured0.bin" + } + ], + "extensionsUsed": [ + "KHR_materials_pbrSpecularGlossiness" + ] +} diff --git a/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured0.bin b/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/BoxTextured0.bin new file mode 100644 index 0000000000000000000000000000000000000000..d2a73551f9456732c7ca85b9d70895f5d9ab8171 GIT binary patch literal 840 zcma)4TMmLS5FAugR8)K(r@fgRg`3F9)Sc2X4N+i|$-K6U9|D@%NdZH8sMCdXhgs?; z>8CE)+Yvq1hwmphbb0bSz9n3Qv{?B+?(fNy1()2HW=AbfwKB4_dS!i9PnJ%1er4>H zi~S!ZIHJM{XG4VxuDIcDxZ(qtxm=$B literal 0 HcmV?d00001 diff --git a/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/CesiumLogoFlat.png b/test/models/glTF2/BoxTextured-glTF-pbrSpecularGlossiness/CesiumLogoFlat.png new file mode 100644 index 0000000000000000000000000000000000000000..45d502ed2610974189307ad542b05b70df14abd4 GIT binary patch literal 2433 zcmV-{34Zp8P)_WXpT zT&KN^W{xz{-NS2=JMQx8=kogA?fBT~_si45y2_@+&$ZLn&C25R(B<{I+3>&4t;yH7 z=k4F%=+?sB^4sOl<>lpUt>3WB;eWT|mc!+SyX3^v-*2zrrpV)YwBmHJ;knS^%huk~ z+1}aP+>XEHoyX^<&g`+&?tmWZ761SUi%CR5RCr$O)Jt;2APhy(Z08|~KgQ|(FDyew zsp=R5YT~nlTY~rX-#i|V$K!G2Wc)mlWh6Gnqti<0l$mFCVfs>HcWU= zU}N>)$8f~oL^fIRfxO@v@>XXMp-0e`br?ZEGI~bCo)PzOtm9az;!cu|<&TbVw_ui5 zwW3|B5@D2()+~cyB$>Y!g)thJ$B!@Lz8kOX+jvq7OjAoBdRRn8M8L$qkQNB4TV7-vHO9JAJQ8q${ zXt%jig;B>Q5c8=c(DHB(vwdB7l^b%9eG_ZB*eh` zKiF!esNH0e)(Q5M{vNJS(>_P z7E8|>Y{ee=q-j zwe7~}bCKVdc&GAaaxy)wQaF3r-P1o;Nh@5_%Rq_QCc}p;>PT)Ya7yW!QtBIp3$O8J zJ-5$rM{OvXImeYE+BRq^+^17@2G@bfcef4nAgl2iHB$;h1ulz}PCmyM7K*Pgam#Cz zbh0@h|EGo$Hx)4kqh8}{YNEJY4eox2vh)a7Aw*_VJcEQL)XA+soKt#8avR94FA)<0 zsg_paDI8oR4sj0WmIU>IQ^k(^v-HmLNUj*xHHJLfb^%}CnhO;skj$V~R0982UN0V=@%S|H5 z;uJ7>9bD|Rm4pZ56i4zoJ=IZa8&Ggv;Yi#>l7j|^50U~WW>Z$`E;0opoG?qS^dksQ zV*JN6Byf8NnFb-wcoF={DLg80PPJyG^kph>&J*9j`iGBTGfe0-wbPcVh6NAbUJ%ZN ztdz1$s|pqw@5&|N%z~zWqbgGr!=~g;l5oa2DkYzHjCdO%3wKgVF4HrZkUROUOf{~M z=Cn_G^Ob30e|2SgbvihOvrJ5;rPOP`dC$3%MtH~4JMp?6`=xmg(XZ1+9^S3ggNITr z$+W~p3hpEhXEc=JYmc?*+v(sg6Nl?7#q*vFq19=}GEq2@Hg>1zQ%$CBpS~2rnOS0& za#N}u@Vq;9lFP*8^xnIcOx^3#he|jz7F=WQ9y??bb?O_=m?Mn?1{t5-UBdr(x8okAfglRt)6NvGbb(TUq6aG6C1`*| zdzTZx{}(gHL?W~88yxWWH+fCNyxk_-f-<2>$;n?PZAF*f9N|EitF7($1=`4kbj+RB zq%{6;d!?4m^<^4zw-2Q39pSgH)E<;c>yvhHG!>eH7p~#y7M}N7(e^O;iAL+mBRC0FBn==#2$yIg$lQO zx(rK>#?!52%-DnNO}C99+-~-4tJF!&d&yW%5pOT9a4Za>mBRBr4~%6L^Xd%8MuwAt zW*I9JR#H@RWw^gmlAPefWm2)lBSjsq4)<3oIiREU(w1y^C$V`S3HLi!D*DZS^HKhB z-YaTx-Y~vTY~lV&rRnC+#&}!`n0Hhst+)K~?bp~HKBXYsQz>P#*{iFndB=5%t3}eu zyJaE*!X5h0647#1nX*kzZ9aFxt9r&pTWVEERI0)~!d3uv zx`oUh2o9eP_tNLr*oYmY2Zk@r_@=_57K0U=t*JxdSRZ`rgl9iokOGFUU25j#hC9S?^-C5lI_BP0Qt)<=X(jOL|W7pUt>r>A!o%X z@co2A{I5!eGgi>%WLzbR5@frDOy5;+oAY(11Qoxd=UJcsgKAsLS-P7pesWtkxv|1P z5P(Z6S#BfSAYj-4UhrO&oM7&Mv@BT$2KrjZJwk9{`B+pT8DmQDz@><>-+!L z>tEMSJWt_oVwj8Nd3MGDj!&X+h~abS8HX(O#RO^i6gmL41?6#A&u5qfhs~Og;BbQH z6ErwO49jO{oU+wZ5WAM8GlG-C#KNsOZL4PUc5ly;HG&U_dCcFnj?3HOLmf+{@G(`* zW72SF|pWh(6ha+1#Q zUegee-PT&(#yCL<-ocN&gH|ibA?|uuzwO#F3cw%?h2bn>D#Qi@U7{I!2=9Ni&V_=s zT@><7S%1j*5*tj$sQk!EVlXA6{mT=^!U_{Z0U#c|HIOG8+R)*Q8CE)+Yvq1hwmphbb0bSz9n3Qv{?B+?(fNy1()2HW=AbfwKB4_dS!i9PnJ%1er4>H zi~S!ZIHJM{XG4VxuDIcDxZ(qtxm=$B literal 0 HcmV?d00001 diff --git a/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured0.vert b/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured0.vert new file mode 100644 index 000000000..05d5e82c2 --- /dev/null +++ b/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured0.vert @@ -0,0 +1,17 @@ +precision highp float; +uniform mat4 u_modelViewMatrix; +uniform mat4 u_projectionMatrix; +uniform mat3 u_normalMatrix; +attribute vec3 a_position; +varying vec3 v_position; +attribute vec3 a_normal; +varying vec3 v_normal; +attribute vec2 a_texcoord0; +varying vec2 v_texcoord0; +void main(void) { + vec4 pos = u_modelViewMatrix * vec4(a_position,1.0); + v_position = pos.xyz; + gl_Position = u_projectionMatrix * pos; + v_normal = u_normalMatrix * a_normal; + v_texcoord0 = a_texcoord0; +} diff --git a/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured1.frag b/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured1.frag new file mode 100644 index 000000000..3850980a0 --- /dev/null +++ b/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/BoxTextured1.frag @@ -0,0 +1,29 @@ +precision highp float; +uniform sampler2D u_diffuse; +uniform vec4 u_specular; +uniform float u_shininess; +uniform float u_transparency; +varying vec3 v_position; +varying vec3 v_normal; +varying vec2 v_texcoord0; +void main(void) { + vec3 normal = normalize(v_normal); + vec4 diffuse = texture2D(u_diffuse, v_texcoord0); + vec3 diffuseLight = vec3(0.0, 0.0, 0.0); + vec3 specular = u_specular.rgb; + vec3 specularLight = vec3(0.0, 0.0, 0.0); + vec3 ambient = diffuse.rgb; + vec3 viewDir = -normalize(v_position); + vec3 ambientLight = vec3(0.0, 0.0, 0.0); + ambientLight += vec3(0.2, 0.2, 0.2); + vec3 l = vec3(0.0, 0.0, 1.0); + diffuseLight += vec3(1.0, 1.0, 1.0) * max(dot(normal, l), 0.); + vec3 reflectDir = reflect(-l, normal); + float specularIntensity = max(0., pow(max(dot(reflectDir, viewDir), 0.), u_shininess)); + specularLight += vec3(1.0, 1.0, 1.0) * specularIntensity; + vec3 color = vec3(0.0, 0.0, 0.0); + color += diffuse.rgb * diffuseLight; + color += specular * specularLight; + color += ambient * ambientLight; + gl_FragColor = vec4(color * diffuse.a * u_transparency, diffuse.a * u_transparency); +} diff --git a/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/CesiumLogoFlat.png b/test/models/glTF2/BoxTextured-glTF-techniqueWebGL/CesiumLogoFlat.png new file mode 100644 index 0000000000000000000000000000000000000000..45d502ed2610974189307ad542b05b70df14abd4 GIT binary patch literal 2433 zcmV-{34Zp8P)_WXpT zT&KN^W{xz{-NS2=JMQx8=kogA?fBT~_si45y2_@+&$ZLn&C25R(B<{I+3>&4t;yH7 z=k4F%=+?sB^4sOl<>lpUt>3WB;eWT|mc!+SyX3^v-*2zrrpV)YwBmHJ;knS^%huk~ z+1}aP+>XEHoyX^<&g`+&?tmWZ761SUi%CR5RCr$O)Jt;2APhy(Z08|~KgQ|(FDyew zsp=R5YT~nlTY~rX-#i|V$K!G2Wc)mlWh6Gnqti<0l$mFCVfs>HcWU= zU}N>)$8f~oL^fIRfxO@v@>XXMp-0e`br?ZEGI~bCo)PzOtm9az;!cu|<&TbVw_ui5 zwW3|B5@D2()+~cyB$>Y!g)thJ$B!@Lz8kOX+jvq7OjAoBdRRn8M8L$qkQNB4TV7-vHO9JAJQ8q${ zXt%jig;B>Q5c8=c(DHB(vwdB7l^b%9eG_ZB*eh` zKiF!esNH0e)(Q5M{vNJS(>_P z7E8|>Y{ee=q-j zwe7~}bCKVdc&GAaaxy)wQaF3r-P1o;Nh@5_%Rq_QCc}p;>PT)Ya7yW!QtBIp3$O8J zJ-5$rM{OvXImeYE+BRq^+^17@2G@bfcef4nAgl2iHB$;h1ulz}PCmyM7K*Pgam#Cz zbh0@h|EGo$Hx)4kqh8}{YNEJY4eox2vh)a7Aw*_VJcEQL)XA+soKt#8avR94FA)<0 zsg_paDI8oR4sj0WmIU>IQ^k(^v-HmLNUj*xHHJLfb^%}CnhO;skj$V~R0982UN0V=@%S|H5 z;uJ7>9bD|Rm4pZ56i4zoJ=IZa8&Ggv;Yi#>l7j|^50U~WW>Z$`E;0opoG?qS^dksQ zV*JN6Byf8NnFb-wcoF={DLg80PPJyG^kph>&J*9j`iGBTGfe0-wbPcVh6NAbUJ%ZN ztdz1$s|pqw@5&|N%z~zWqbgGr!=~g;l5oa2DkYzHjCdO%3wKgVF4HrZkUROUOf{~M z=Cn_G^Ob30e|2SgbvihOvrJ5;rPOP`dC$3%MtH~4JMp?6`=xmg(XZ1+9^S3ggNITr z$+W~p3hpEhXEc=JYmc?*+v(sg6Nl?7#q*vFq19=}GEq2@Hg>1zQ%$CBpS~2rnOS0& za#N}u@Vq;9lFP*8^xnIcOx^3#he|jz7F=WQ9y??bb?O_=m?Mn?1{t5-UBdr(x8okAfglRt)6NvGbb(TUq6aG6C1`*| zdzTZx{}(gHL?W~88yxWWH+fCNyxk_-f-<2>$;n?PZAF*f9N|EitF7($1=`4kbj+RB zq%{6;d!?4m^<^4zw-2Q39pSgH)E<;c>yvhHG!>eH7p~#y7M}N7(e^O;iAL+mBRC0FBn==#2$yIg$lQO zx(rK>#?!52%-DnNO}C99+-~-4tJF!&d&yW%5pOT9a4Za>mBRBr4~%6L^Xd%8MuwAt zW*I9JR#H@RWw^gmlAPefWm2)lBSjsq4)<3oIiREU(w1y^C$V`S3HLi!D*DZS^HKhB z-YaTx-Y~vTY~lV&rRnC+#&}!`n0Hhst+)K~?bp~HKBXYsQz>P#*{iFndB=5%t3}eu zyJaE*!X5h0647#1nX*kzZ9aFxt9r&pTWVEERI0)~!d3uv zx`oUh2o9eP_tNLr*oYmY2Zk@r_@=_57K0U=t*JxdSRZ`rgl9iokOGFUU25j#hC9S?^-C5lI_BP0Qt)<=X(jOL|W7pUt>r>A!o%X z@co2A{I5!eGgi>%WLzbR5@frDOy5;+oAY(11Qoxd=UJcsgKAsLS-P7pesWtkxv|1P z5P(Z6S#BfSAYj-4UhrO&oM7&Mv@BT$2KrjZJwk9{`B+pT8DmQDz@><>-+!L z>tEMSJWt_oVwj8Nd3MGDj!&X+h~abS8HX(O#RO^i6gmL41?6#A&u5qfhs~Og;BbQH z6ErwO49jO{oU+wZ5WAM8GlG-C#KNsOZL4PUc5ly;HG&U_dCcFnj?3HOLmf+{@G(`* zW72SF|pWh(6ha+1#Q zUegee-PT&(#yCL<-ocN&gH|ibA?|uuzwO#F3cw%?h2bn>D#Qi@U7{I!2=9Ni&V_=s zT@><7S%1j*5*tj$sQk!EVlXA6{mT=^!U_{Z0U#c|HIOG8+R)*Q8CE)+Yvq1hwmphbb0bSz9n3Qv{?B+?(fNy1()2HW=AbfwKB4_dS!i9PnJ%1er4>H zi~S!ZIHJM{XG4VxuDIcDxZ(qtxm=$B literal 0 HcmV?d00001 diff --git a/test/models/glTF2/BoxTextured-glTF/CesiumLogoFlat.png b/test/models/glTF2/BoxTextured-glTF/CesiumLogoFlat.png new file mode 100644 index 0000000000000000000000000000000000000000..45d502ed2610974189307ad542b05b70df14abd4 GIT binary patch literal 2433 zcmV-{34Zp8P)_WXpT zT&KN^W{xz{-NS2=JMQx8=kogA?fBT~_si45y2_@+&$ZLn&C25R(B<{I+3>&4t;yH7 z=k4F%=+?sB^4sOl<>lpUt>3WB;eWT|mc!+SyX3^v-*2zrrpV)YwBmHJ;knS^%huk~ z+1}aP+>XEHoyX^<&g`+&?tmWZ761SUi%CR5RCr$O)Jt;2APhy(Z08|~KgQ|(FDyew zsp=R5YT~nlTY~rX-#i|V$K!G2Wc)mlWh6Gnqti<0l$mFCVfs>HcWU= zU}N>)$8f~oL^fIRfxO@v@>XXMp-0e`br?ZEGI~bCo)PzOtm9az;!cu|<&TbVw_ui5 zwW3|B5@D2()+~cyB$>Y!g)thJ$B!@Lz8kOX+jvq7OjAoBdRRn8M8L$qkQNB4TV7-vHO9JAJQ8q${ zXt%jig;B>Q5c8=c(DHB(vwdB7l^b%9eG_ZB*eh` zKiF!esNH0e)(Q5M{vNJS(>_P z7E8|>Y{ee=q-j zwe7~}bCKVdc&GAaaxy)wQaF3r-P1o;Nh@5_%Rq_QCc}p;>PT)Ya7yW!QtBIp3$O8J zJ-5$rM{OvXImeYE+BRq^+^17@2G@bfcef4nAgl2iHB$;h1ulz}PCmyM7K*Pgam#Cz zbh0@h|EGo$Hx)4kqh8}{YNEJY4eox2vh)a7Aw*_VJcEQL)XA+soKt#8avR94FA)<0 zsg_paDI8oR4sj0WmIU>IQ^k(^v-HmLNUj*xHHJLfb^%}CnhO;skj$V~R0982UN0V=@%S|H5 z;uJ7>9bD|Rm4pZ56i4zoJ=IZa8&Ggv;Yi#>l7j|^50U~WW>Z$`E;0opoG?qS^dksQ zV*JN6Byf8NnFb-wcoF={DLg80PPJyG^kph>&J*9j`iGBTGfe0-wbPcVh6NAbUJ%ZN ztdz1$s|pqw@5&|N%z~zWqbgGr!=~g;l5oa2DkYzHjCdO%3wKgVF4HrZkUROUOf{~M z=Cn_G^Ob30e|2SgbvihOvrJ5;rPOP`dC$3%MtH~4JMp?6`=xmg(XZ1+9^S3ggNITr z$+W~p3hpEhXEc=JYmc?*+v(sg6Nl?7#q*vFq19=}GEq2@Hg>1zQ%$CBpS~2rnOS0& za#N}u@Vq;9lFP*8^xnIcOx^3#he|jz7F=WQ9y??bb?O_=m?Mn?1{t5-UBdr(x8okAfglRt)6NvGbb(TUq6aG6C1`*| zdzTZx{}(gHL?W~88yxWWH+fCNyxk_-f-<2>$;n?PZAF*f9N|EitF7($1=`4kbj+RB zq%{6;d!?4m^<^4zw-2Q39pSgH)E<;c>yvhHG!>eH7p~#y7M}N7(e^O;iAL+mBRC0FBn==#2$yIg$lQO zx(rK>#?!52%-DnNO}C99+-~-4tJF!&d&yW%5pOT9a4Za>mBRBr4~%6L^Xd%8MuwAt zW*I9JR#H@RWw^gmlAPefWm2)lBSjsq4)<3oIiREU(w1y^C$V`S3HLi!D*DZS^HKhB z-YaTx-Y~vTY~lV&rRnC+#&}!`n0Hhst+)K~?bp~HKBXYsQz>P#*{iFndB=5%t3}eu zyJaE*!X5h0647#1nX*kzZ9aFxt9r&pTWVEERI0)~!d3uv zx`oUh2o9eP_tNLr*oYmY2Zk@r_@=_57K0U=t*JxdSRZ`rgl9iokOGFUU25j#hC9S?^-C5lI_BP0Qt)<=X(jOL|W7pUt>r>A!o%X z@co2A{I5!eGgi>%WLzbR5@frDOy5;+oAY(11Qoxd=UJcsgKAsLS-P7pesWtkxv|1P z5P(Z6S#BfSAYj-4UhrO&oM7&Mv@BT$2KrjZJwk9{`B+pT8DmQDz@><>-+!L z>tEMSJWt_oVwj8Nd3MGDj!&X+h~abS8HX(O#RO^i6gmL41?6#A&u5qfhs~Og;BbQH z6ErwO49jO{oU+wZ5WAM8GlG-C#KNsOZL4PUc5ly;HG&U_dCcFnj?3HOLmf+{@G(`* zW72SF|pWh(6ha+1#Q zUegee-PT&(#yCL<-ocN&gH|ibA?|uuzwO#F3cw%?h2bn>D#Qi@U7{I!2=9Ni&V_=s zT@><7S%1j*5*tj$sQk!EVlXA6{mT=^!U_{Z0U#c|HIOG8+R)*Q Date: Wed, 30 Aug 2017 17:25:11 -0400 Subject: [PATCH 096/305] Working read, import, export, and write of gltf2 (pbr) material --- code/glTF2Asset.h | 51 ++++++++------ code/glTF2Asset.inl | 115 +++++++++++++++++++----------- code/glTF2AssetWriter.inl | 145 ++++++++++++++++++++++++++++++++------ code/glTF2Exporter.cpp | 50 +++++++------ code/glTF2Exporter.h | 10 ++- code/glTF2Importer.cpp | 93 +++++++++++++++++------- 6 files changed, 329 insertions(+), 135 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 5d99735b2..509fef115 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -133,7 +133,6 @@ namespace glTF2 struct Light; struct Skin; - // Vec/matrix types, as raw float arrays typedef float (vec3)[3]; typedef float (vec4)[4]; @@ -671,20 +670,30 @@ namespace glTF2 inline void SetData(uint8_t* data, size_t length, Asset& r); }; - struct ColorProperty - { - union { - vec4, vec3 - } color; - } - - //! Holds a material property that can be a texture or a color (fallback for glTF 1) - struct FallbackTexProperty + struct TextureInfo { Ref texture; - ColorProperty color; + unsigned int index; + unsigned int texCoord = 0; }; + struct NormalTextureInfo : TextureInfo + { + float scale = 1; + }; + + struct OcclusionTextureInfo : TextureInfo + { + float strength = 1; + }; + + //! Holds a material property that can be a texture or a color (fallback for glTF 1) + /*struct FallbackTexProperty + { + Ref texture; + vec4 color; + };*/ + //! The material appearance of a primitive. struct Material : public Object { @@ -702,26 +711,26 @@ namespace glTF2 }; //PBR metallic roughness properties - ColorProperty baseColor; - Ref baseColorTexture; - Ref metallicRoughnessTexture; + vec4 baseColorFactor; + TextureInfo baseColorTexture; + TextureInfo metallicRoughnessTexture; float metallicFactor; float roughnessFactor; //other basic material properties - Ref normalTexture; - Ref occlusionTexture; - Ref emissiveTexture; - ColorProperty emissiveFactor; + NormalTextureInfo normalTexture; + OcclusionTextureInfo occlusionTexture; + TextureInfo emissiveTexture; + vec3 emissiveFactor; std::string alphaMode; float alphaCutoff; bool doubleSided; //fallback material properties (compatible with non-pbr defintions) - FallbackTexProperty diffuse; - FallbackTexProperty emissive; + /*FallbackTexProperty diffuse; + FallbackTexProperty emission; FallbackTexProperty specular; - Ref normal; + Ref normal;*/ Technique technique; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index bf8cb5a36..94858044d 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -128,6 +128,12 @@ namespace { return (it != val.MemberEnd() && it->value.IsString()) ? &it->value : 0; } + inline Value* FindNumber(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd() && it->value.IsNumber()) ? &it->value : 0; + } + inline Value* FindUInt(Value& val, const char* id) { Value::MemberIterator it = val.FindMember(id); @@ -225,7 +231,7 @@ Ref LazyDict::Retrieve(unsigned int i) } T* inst = new T(); - inst->id = std::string(mDictId) + "[" + std::to_string(i) + "]"; + inst->id = std::string(mDictId) + "_" + std::to_string(i); inst->oIndex = i; ReadMember(obj, "name", inst->name); inst->Read(obj, mAsset); @@ -706,15 +712,42 @@ inline void Texture::Read(Value& obj, Asset& r) } namespace { - inline void ReadMaterialProperty(Asset& r, Value& vals, const char* propName, TexProperty& out) + inline void SetTextureProperties(Asset& r, Value* prop, TextureInfo& out) + { + if (Value* index = FindUInt(*prop, "index")) { + out.texture = r.textures.Retrieve(index->GetUint()); + } + + if (Value* texcoord = FindUInt(*prop, "texCoord")) { + out.texCoord = texcoord->GetUint(); + } + } + + inline void ReadTextureProperty(Asset& r, Value& vals, const char* propName, TextureInfo& out) { - //@TODO: update this format if (Value* prop = FindMember(vals, propName)) { - if (prop->IsUint()) { - out.texture = r.textures.Retrieve(prop->GetUint()); + SetTextureProperties(r, prop, out); + } + } + + inline void ReadTextureProperty(Asset& r, Value& vals, const char* propName, NormalTextureInfo& out) + { + if (Value* prop = FindMember(vals, propName)) { + SetTextureProperties(r, prop, out); + + if (Value* scale = FindNumber(*prop, "scale")) { + out.scale = scale->GetDouble(); } - else { - ReadValue(*prop, out.color); + } + } + + inline void ReadTextureProperty(Asset& r, Value& vals, const char* propName, OcclusionTextureInfo& out) + { + if (Value* prop = FindMember(vals, propName)) { + SetTextureProperties(r, prop, out); + + if (Value* strength = FindNumber(*prop, "strength")) { + out.strength = strength->GetDouble(); } } } @@ -724,33 +757,24 @@ inline void Material::Read(Value& material, Asset& r) { SetDefaults(); - if (Value* values = FindObject(material, "values")) { - - - ReadMember(*values, "transparency", transparency); - } - if (Value* values = FindObject(material, "pbrMetallicRoughness")) { - //pbr - ReadMaterialProperty(r, *values, "baseColorFactor", this->baseColor); - ReadMaterialProperty(r, *values, "baseColorTexture", this->baseColorTexture); - - //non-pbr fallback - ReadMaterialProperty(r, *values, "baseColorFactor", this->diffuse); - ReadMaterialProperty(r, *values, "baseColorTexture", this->diffuse); - - ReadMember(*values, "metallicFactor", metallicFactor); + ReadMember(*values, "baseColorFactor", this->baseColorFactor); + ReadTextureProperty(r, *values, "baseColorTexture", this->baseColorTexture); + ReadTextureProperty(r, *values, "metallicRoughnessTexture", this->metallicRoughnessTexture); + ReadMember(*values, "metallicFactor", this->metallicFactor); + ReadMember(*values, "roughnessFactor", this->roughnessFactor); } - ReadMaterialProperty(r, *values, "normalTexture", this->normalTexture); - ReadMaterialProperty(r, *values, "normalTexture", this->normal); - ReadMaterialProperty(r, *values, "occlusionTexture", this->occlusionTexture); - ReadMaterialProperty(r, *values, "emissiveTexture", this->emissiveTexture); - ReadMember(*values, "metallicFactor", emissiveFactor); + ReadTextureProperty(r, material, "normalTexture", this->normalTexture); + ReadTextureProperty(r, material, "occlusionTexture", this->occlusionTexture); + ReadTextureProperty(r, material, "emissiveTexture", this->emissiveTexture); + ReadMember(material, "emissiveFactor", this->emissiveFactor); - ReadMember(material, "doubleSided", doubleSided); + ReadMember(material, "doubleSided", this->doubleSided); + ReadMember(material, "alphaMode", this->alphaMode); + ReadMember(material, "alphaCutoff", this->alphaCutoff); - if (Value* extensions = FindObject(material, "extensions")) { + /* if (Value* extensions = FindObject(material, "extensions")) { if (r.extensionsUsed.KHR_materials_common) { if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { if (Value* tnq = FindString(*ext, "technique")) { @@ -762,9 +786,9 @@ inline void Material::Read(Value& material, Asset& r) } if (Value* values = FindObject(*ext, "values")) { - ReadMaterialProperty(r, *values, "ambient", this->ambient); - ReadMaterialProperty(r, *values, "diffuse", this->diffuse); - ReadMaterialProperty(r, *values, "specular", this->specular); + ReadTextureProperty(r, *values, "ambient", this->ambient); + ReadTextureProperty(r, *values, "diffuse", this->diffuse); + ReadTextureProperty(r, *values, "specular", this->specular); ReadMember(*values, "doubleSided", doubleSided); ReadMember(*values, "transparent", transparent); @@ -773,25 +797,28 @@ inline void Material::Read(Value& material, Asset& r) } } } - } + } */ } namespace { void SetVector(vec4& v, float x, float y, float z, float w) { v[0] = x; v[1] = y; v[2] = z; v[3] = w; } + + void SetVector(vec3& v, float x, float y, float z) + { v[0] = x; v[1] = y; v[2] = z; } } inline void Material::SetDefaults() { - SetVector(ambient.color, 0, 0, 0, 1); - SetVector(diffuse.color, 0, 0, 0, 1); - SetVector(specular.color, 0, 0, 0, 1); - SetVector(emission.color, 0, 0, 0, 1); + //pbr materials + SetVector(baseColorFactor, 1, 1, 1, 1); + SetVector(emissiveFactor, 0, 0, 0); + metallicFactor = 1.0; + roughnessFactor = 1.0; + alphaMode = "OPAQUE"; + alphaCutoff = 0.5; doubleSided = false; - transparent = false; - transparency = 1.0; - shininess = 0.0; technique = Technique_undefined; } @@ -833,6 +860,10 @@ namespace { inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) { + if (Value* name = FindMember(pJSON_Object, "name")) { + this->name = name->GetString(); + } + /****************** Mesh primitives ******************/ if (Value* primitives = FindArray(pJSON_Object, "primitives")) { this->primitives.resize(primitives->Size()); @@ -1055,7 +1086,7 @@ inline void Node::Read(Value& obj, Asset& r) // TODO load "skeletons", "skin", "jointName" - if (Value* extensions = FindObject(obj, "extensions")) { + /*if (Value* extensions = FindObject(obj, "extensions")) { if (r.extensionsUsed.KHR_materials_common) { if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { @@ -1065,7 +1096,7 @@ inline void Node::Read(Value& obj, Asset& r) } } - } + }*/ } inline void Scene::Read(Value& obj, Asset& r) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index ed8ee80ea..a939e38d6 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -72,6 +72,12 @@ namespace glTF2 { return val; } + inline Value& MakeValue(Value& val, float r, MemoryPoolAllocator<>& al) { + val.SetDouble(r); + + return val; + } + template inline void AddRefsVector(Value& obj, const char* fieldId, std::vector< Ref >& v, MemoryPoolAllocator<>& al) { if (v.empty()) return; @@ -196,51 +202,148 @@ namespace glTF2 { } namespace { - inline void WriteTex(Value& obj, Ref texture, const char* propName, MemoryPoolAllocator<>& al) + inline void SetTexBasic(TextureInfo t, Value& tex, MemoryPoolAllocator<>& al) { - if (texture) { + tex.SetObject(); + tex.AddMember("index", t.texture->index, al); + + if (t.texCoord != 0) { + tex.AddMember("texCoord", t.texCoord, al); + } + } + + inline void WriteTex(Value& obj, TextureInfo t, const char* propName, MemoryPoolAllocator<>& al) + { + + if (t.texture) { Value tex; - tex.SetObject(); - tex.AddMember("index", texture->index, al); + + SetTexBasic(t, tex, al); + obj.AddMember(StringRef(propName), tex, al); } } - inline void WriteColorOrTex(Value& obj, TexProperty& prop, const char* propName, MemoryPoolAllocator<>& al) + inline void WriteTex(Value& obj, NormalTextureInfo t, const char* propName, MemoryPoolAllocator<>& al) { - WriteTex(obj, prop.texture, propName, al); - if (!prop.texture) { - Value col; - obj.AddMember(StringRef(propName), MakeValue(col, prop.color, al), al); + + if (t.texture) { + Value tex; + + SetTexBasic(t, tex, al); + + if (t.scale != 1) { + tex.AddMember("scale", t.scale, al); + } + + obj.AddMember(StringRef(propName), tex, al); } } + + inline void WriteTex(Value& obj, OcclusionTextureInfo t, const char* propName, MemoryPoolAllocator<>& al) + { + + if (t.texture) { + Value tex; + + SetTexBasic(t, tex, al); + + if (t.strength != 1) { + tex.AddMember("strength", t.strength, al); + } + + obj.AddMember(StringRef(propName), tex, al); + } + } + + template + inline void WriteVec(Value& obj, float(&prop)[N], const char* propName, MemoryPoolAllocator<>& al) + { + Value arr; + obj.AddMember(StringRef(propName), MakeValue(arr, prop, al), al); + } + + template + inline void WriteVec(Value& obj, float(&prop)[N], const char* propName, float(&defaultVal)[N], MemoryPoolAllocator<>& al) + { + if (!std::equal(std::begin(prop), std::end(prop), std::begin(defaultVal))) { + WriteVec(obj, prop, propName, al); + } + } + + inline void WriteFloat(Value& obj, float prop, const char* propName, MemoryPoolAllocator<>& al) + { + Value num; + obj.AddMember(StringRef(propName), MakeValue(num, prop, al), al); + } } inline void Write(Value& obj, Material& m, AssetWriter& w) { - if (m.transparent) { - obj.AddMember("alphaMode", "BLEND", w.mAl); + if (!m.name.empty()) { + obj.AddMember("name", m.name, w.mAl); } - Value v; + Value pbrMetallicRoughness; + pbrMetallicRoughness.SetObject(); + { + WriteTex(pbrMetallicRoughness, m.baseColorTexture, "baseColorTexture", w.mAl); + WriteTex(pbrMetallicRoughness, m.metallicRoughnessTexture, "metallicRoughnessTexture", w.mAl); + + //@TODO: define this as a constant? + vec4 defaultEmissiveFactor = {1, 1, 1, 1}; + WriteVec(pbrMetallicRoughness, m.baseColorFactor, "baseColorFactor", defaultEmissiveFactor, w.mAl); + + if (m.metallicFactor != 1) { + WriteFloat(pbrMetallicRoughness, m.metallicFactor, "metallicFactor", w.mAl); + } + + if (m.roughnessFactor != 1) { + WriteFloat(pbrMetallicRoughness, m.roughnessFactor, "roughnessFactor", w.mAl); + } + } + + if (pbrMetallicRoughness.MemberCount() > 0) { + obj.AddMember("pbrMetallicRoughness", pbrMetallicRoughness, w.mAl); + } + + WriteTex(obj, m.normalTexture, "normalTexture", w.mAl); + WriteTex(obj, m.emissiveTexture, "emissiveTexture", w.mAl); + WriteTex(obj, m.occlusionTexture, "occlusionTexture", w.mAl); + + //@TODO: define this as a constant? + vec3 defaultEmissiveFactor = {0, 0, 0}; + WriteVec(obj, m.emissiveFactor, "emissiveFactor", defaultEmissiveFactor, w.mAl); + + if (m.alphaCutoff != 0.5) { + WriteFloat(obj, m.alphaCutoff, "alphaCutoff", w.mAl); + } + + if (m.alphaMode != "OPAQUE") { + obj.AddMember("alphaMode", m.alphaMode, w.mAl); + } + + if (m.doubleSided) { + obj.AddMember("doubleSided", m.doubleSided, w.mAl); + } + + /*Value v; v.SetObject(); { if (m.transparent && !m.diffuse.texture) { m.diffuse.color[3] = m.transparency; } - WriteColorOrTex(v, m.ambient, m.ambient.texture ? "ambientTexture" : "ambientFactor", w.mAl); - WriteColorOrTex(v, m.diffuse, m.diffuse.texture ? "diffuseTexture" : "diffuseFactor", w.mAl); - WriteColorOrTex(v, m.specular, m.specular.texture ? "specularTexture" : "specularFactor", w.mAl); - WriteColorOrTex(v, m.emission, m.emission.texture ? "emissionTexture" : "emissionFactor", w.mAl); + WriteVecOrTex(v, m.ambient, m.ambient.texture ? "ambientTexture" : "ambientFactor", w.mAl); + WriteVecOrTex(v, m.diffuse, m.diffuse.texture ? "diffuseTexture" : "diffuseFactor", w.mAl); + WriteVecOrTex(v, m.specular, m.specular.texture ? "specularTexture" : "specularFactor", w.mAl); + WriteVecOrTex(v, m.emission, m.emission.texture ? "emissionTexture" : "emissionFactor", w.mAl); v.AddMember("shininessFactor", m.shininess, w.mAl); } v.AddMember("type", "commonPhong", w.mAl); Value ext; ext.SetObject(); ext.AddMember("KHR_materials_common", v, w.mAl); - obj.AddMember("extensions", ext, w.mAl); - - WriteTex(obj, m.normal, "normalTexture", w.mAl); + obj.AddMember("extensions", ext, w.mAl);*/ } namespace { @@ -263,8 +366,6 @@ namespace glTF2 { inline void Write(Value& obj, Mesh& m, AssetWriter& w) { - /********************* Name **********************/ - obj.AddMember("name", m.name, w.mAl); /**************** Mesh extensions ****************/ if(m.Extension.size() > 0) @@ -617,7 +718,7 @@ namespace glTF2 { exts.PushBack(StringRef("KHR_binary_glTF"), mAl); // This is used to export common materials with GLTF 2. - exts.PushBack(StringRef("KHR_materials_common"), mAl); + //exts.PushBack(StringRef("KHR_materials_common"), mAl); } if (!exts.Empty()) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 68f2f4420..f59c280f1 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -234,12 +234,6 @@ inline Ref ExportData(Asset& a, std::string& meshName, Ref& bu return acc; } -namespace { - void GetMatScalar(const aiMaterial* mat, float& val, const char* propName, int type, int idx) { - if (mat->Get(propName, type, idx, val) == AI_SUCCESS) {} - } -} - void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture) { std::string samplerId = mAsset->FindUniqueID("", "sampler"); @@ -286,11 +280,11 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture) texture->sampler->minFilter = SamplerMinFilter_Linear_Mipmap_Linear; } -void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt) +void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt, unsigned int slot = 0) { aiString tex; if (mat->GetTextureCount(tt) > 0) { - if (mat->Get(AI_MATKEY_TEXTURE(tt, 0), tex) == AI_SUCCESS) { + if (mat->Get(AI_MATKEY_TEXTURE(tt, slot), tex) == AI_SUCCESS) { std::string path = tex.C_Str(); if (path.size() > 0) { @@ -332,15 +326,21 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe } } -void glTF2Exporter::GetMatColorOrTex(const aiMaterial* mat, TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt) +void glTF2Exporter::GetMatColor(const aiMaterial* mat, vec4& prop, const char* propName, int type, int idx) { aiColor4D col; if (mat->Get(propName, type, idx, col) == AI_SUCCESS) { - prop.color[0] = col.r; prop.color[1] = col.g; prop.color[2] = col.b; prop.color[3] = col.a; + prop[0] = col.r; prop[1] = col.g; prop[2] = col.b; prop[3] = col.a; } - GetMatTex(mat, prop.texture, tt); } +void glTF2Exporter::GetMatColor(const aiMaterial* mat, vec3& prop, const char* propName, int type, int idx) +{ + aiColor3D col; + if (mat->Get(propName, type, idx, col) == AI_SUCCESS) { + prop[0] = col.r; prop[1] = col.g; prop[2] = col.b; + } +} void glTF2Exporter::ExportMaterials() { @@ -348,7 +348,6 @@ void glTF2Exporter::ExportMaterials() for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { const aiMaterial* mat = mScene->mMaterials[i]; - std::string name; if (mat->Get(AI_MATKEY_NAME, aiName) == AI_SUCCESS) { name = aiName.C_Str(); @@ -357,15 +356,20 @@ void glTF2Exporter::ExportMaterials() Ref m = mAsset->materials.Create(name); - GetMatColorOrTex(mat, m->ambient, AI_MATKEY_COLOR_AMBIENT, aiTextureType_AMBIENT); - GetMatColorOrTex(mat, m->diffuse, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_DIFFUSE); - GetMatColorOrTex(mat, m->specular, AI_MATKEY_COLOR_SPECULAR, aiTextureType_SPECULAR); - GetMatColorOrTex(mat, m->emission, AI_MATKEY_COLOR_EMISSIVE, aiTextureType_EMISSIVE); - GetMatTex(mat, m->normal, aiTextureType_NORMALS); + GetMatTex(mat, m->baseColorTexture.texture, aiTextureType_DIFFUSE); + GetMatTex(mat, m->metallicRoughnessTexture.texture, aiTextureType_UNKNOWN, 0);//get unknown slot + GetMatTex(mat, m->emissiveTexture.texture, aiTextureType_EMISSIVE); + GetMatTex(mat, m->normalTexture.texture, aiTextureType_NORMALS); + GetMatTex(mat, m->occlusionTexture.texture, aiTextureType_LIGHTMAP); - m->transparent = mat->Get(AI_MATKEY_OPACITY, m->transparency) == aiReturn_SUCCESS && m->transparency != 1.0; + GetMatColor(mat, m->baseColorFactor, AI_MATKEY_COLOR_DIFFUSE); + GetMatColor(mat, m->emissiveFactor, AI_MATKEY_COLOR_EMISSIVE); - GetMatScalar(mat, m->shininess, AI_MATKEY_SHININESS); + mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); + mat->Get("$mat.gltf.alphaCutoff", 0, 0, m->alphaCutoff); + mat->Get("$mat.gltf.metallicFactor", 0, 0, m->metallicFactor); + mat->Get("$mat.gltf.roughnessFactor", 0, 0, m->roughnessFactor); + mat->Get("$mat.gltf.alphaMode", 0, 0, m->alphaMode); } } @@ -566,13 +570,17 @@ void glTF2Exporter::ExportMeshes() DefaultLogger::get()->warn("GLTF: can not use Open3DGC-compression: " + msg); comp_allow = false; - } + } - std::string meshId = mAsset->FindUniqueID(aim->mName.C_Str(), "mesh"); + std::string name = aim->mName.C_Str(); + + std::string meshId = mAsset->FindUniqueID(name, "mesh"); Ref m = mAsset->meshes.Create(meshId); m->primitives.resize(1); Mesh::Primitive& p = m->primitives.back(); + m->name = name; + p.material = mAsset->materials.Get(aim->mMaterialIndex); /******************* Vertices ********************/ diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index 17cc9cfdc..2ba0143bf 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -66,8 +66,13 @@ namespace glTF2 class Asset; struct TexProperty; + struct TextureInfo; struct Node; struct Texture; + + // Vec/matrix types, as raw float arrays + typedef float (vec3)[3]; + typedef float (vec4)[4]; } namespace Assimp @@ -102,8 +107,9 @@ namespace Assimp void WriteBinaryData(IOStream* outfile, std::size_t sceneLength); void GetTexSampler(const aiMaterial* mat, glTF2::Ref texture); - void GetMatTex(const aiMaterial* mat, glTF2::Ref& texture, aiTextureType tt); - void GetMatColorOrTex(const aiMaterial* mat, glTF2::TexProperty& prop, const char* propName, int type, int idx, aiTextureType tt); + void GetMatTex(const aiMaterial* mat, glTF2::Ref& texture, aiTextureType tt, unsigned int slot); + void GetMatColor(const aiMaterial* mat, glTF2::vec4& prop, const char* propName, int type, int idx); + void GetMatColor(const aiMaterial* mat, glTF2::vec3& prop, const char* propName, int type, int idx); void ExportMetadata(); void ExportMaterials(); void ExportMeshes(); diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index bd4f42600..25cb39168 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -141,6 +141,11 @@ static void CopyValue(const glTF2::vec4& v, aiColor3D& out) out.r = v[0]; out.g = v[1]; out.b = v[2]; } +static void CopyValue(const glTF2::vec3& v, aiColor4D& out) +{ + out.r = v[0]; out.g = v[1]; out.b = v[2]; out.a = 1.0; +} + static void CopyValue(const glTF2::vec3& v, aiVector3D& out) { out.x = v[0]; out.y = v[1]; out.z = v[2]; @@ -159,28 +164,56 @@ static void CopyValue(const glTF2::mat4& v, aiMatrix4x4& o) o.a4 = v[12]; o.b4 = v[13]; o.c4 = v[14]; o.d4 = v[15]; } -inline void SetMaterialColorProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TexProperty prop, aiMaterial* mat, - aiTextureType texType, const char* pKey, unsigned int type, unsigned int idx) +inline void SetMaterialColorProperty(Asset& r, vec4& prop, aiMaterial* mat, const char* pKey, unsigned int type, unsigned int idx) { - if (prop.texture) { - if (prop.texture->source) { - aiString uri(prop.texture->source->uri); + aiColor4D col; + CopyValue(prop, col); + mat->AddProperty(&col, 1, pKey, type, idx); +} - int texIdx = embeddedTexIdxs[prop.texture->source.GetIndex()]; - if (texIdx != -1) { // embedded - // setup texture reference string (copied from ColladaLoader::FindFilenameForEffectTexture) - uri.data[0] = '*'; - uri.length = 1 + ASSIMP_itoa10(uri.data + 1, MAXLEN - 1, texIdx); - } +inline void SetMaterialColorProperty(Asset& r, vec3& prop, aiMaterial* mat, const char* pKey, unsigned int type, unsigned int idx) +{ + vec4 prop4; + prop4[0] = prop[0]; + prop4[1] = prop[1]; + prop4[2] = prop[2]; + prop4[3] = 1; + + return SetMaterialColorProperty(r, prop4, mat, pKey, type, idx); +} + +inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType, int texSlot) +{ + if (prop.texture && prop.texture->source) { + aiString uri(prop.texture->source->uri); + + int texIdx = embeddedTexIdxs[prop.texture->source.GetIndex()]; + if (texIdx != -1) { // embedded + // setup texture reference string (copied from ColladaLoader::FindFilenameForEffectTexture) + uri.data[0] = '*'; + uri.length = 1 + ASSIMP_itoa10(uri.data + 1, MAXLEN - 1, texIdx); + } + + if (texSlot < 0) { mat->AddProperty(&uri, _AI_MATKEY_TEXTURE_BASE, texType, 0); } + else { + mat->AddProperty(&uri, AI_MATKEY_TEXTURE(texType, + texSlot)); + } } - else { - aiColor4D col; - CopyValue(prop.color, col); - mat->AddProperty(&col, 1, pKey, type, idx); - } +} + +//import textures that are only supported in pbr contexts +inline void SetMaterialPBRTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, unsigned int texSlot) +{ + return SetMaterialTextureProperty(embeddedTexIdxs, r, prop, mat, aiTextureType_UNKNOWN, texSlot); +} + +inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType) +{ + return SetMaterialTextureProperty(embeddedTexIdxs, r, prop, mat, texType, -1); } void glTF2Importer::ImportMaterials(glTF2::Asset& r) @@ -193,18 +226,23 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) Material& mat = r.materials[i]; - /*if (!mat.name.empty())*/ { - aiString str(mat.id /*mat.name*/); - aimat->AddProperty(&str, AI_MATKEY_NAME); - } + aiString str(mat.id); + aimat->AddProperty(&str, AI_MATKEY_NAME); - SetMaterialColorProperty(embeddedTexIdxs, r, mat.diffuse, aimat, aiTextureType_DIFFUSE, AI_MATKEY_COLOR_DIFFUSE); - SetMaterialColorProperty(embeddedTexIdxs, r, mat.specular, aimat, aiTextureType_SPECULAR, AI_MATKEY_COLOR_SPECULAR); - SetMaterialColorProperty(embeddedTexIdxs, r, mat.ambient, aimat, aiTextureType_AMBIENT, AI_MATKEY_COLOR_AMBIENT); + SetMaterialColorProperty(r, mat.baseColorFactor, aimat, AI_MATKEY_COLOR_DIFFUSE); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.baseColorTexture, aimat, aiTextureType_DIFFUSE); + SetMaterialPBRTextureProperty(embeddedTexIdxs, r, mat.metallicRoughnessTexture, aimat, 0); + aimat->AddProperty(&mat.metallicFactor, 1, "$mat.gltf.metallicFactor"); + aimat->AddProperty(&mat.roughnessFactor, 1, "$mat.gltf.roughnessFactor"); - if (mat.shininess > 0.f) { - aimat->AddProperty(&mat.shininess, 1, AI_MATKEY_SHININESS); - } + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.normalTexture, aimat, aiTextureType_NORMALS); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.occlusionTexture, aimat, aiTextureType_LIGHTMAP); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.emissiveTexture, aimat, aiTextureType_EMISSIVE); + SetMaterialColorProperty(r, mat.emissiveFactor, aimat, AI_MATKEY_COLOR_EMISSIVE); + + aimat->AddProperty(&mat.doubleSided, 1, AI_MATKEY_TWOSIDED); + aimat->AddProperty(&mat.alphaMode, 1, "$mat.gltf.alphaMode"); + aimat->AddProperty(&mat.alphaCutoff, 1, "$mat.gltf.alphaCutoff"); } } @@ -295,7 +333,8 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) aiMesh* aim = new aiMesh(); meshes.push_back(aim); - aim->mName = mesh.id; + aim->mName = mesh.name.empty() ? mesh.id : mesh.name; + if (mesh.primitives.size() > 1) { size_t& len = aim->mName.length; aim->mName.data[len] = '-'; From 0cf69479c30c97a6f4bfeb00ffdceab5749b2b5f Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 30 Aug 2017 21:50:06 -0400 Subject: [PATCH 097/305] Use `!ObjectEmpty()` vs. `MemberCount() > 0` --- code/glTF2AssetWriter.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index a939e38d6..bd16a7023 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -303,7 +303,7 @@ namespace glTF2 { } } - if (pbrMetallicRoughness.MemberCount() > 0) { + if (!pbrMetallicRoughness.ObjectEmpty()) { obj.AddMember("pbrMetallicRoughness", pbrMetallicRoughness, w.mAl); } From 7615a97cd3164d624f3762dd3edf49f6e461a88d Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 30 Aug 2017 23:51:52 -0400 Subject: [PATCH 098/305] Remove redundant function --- code/glTF2Importer.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 25cb39168..b1bbf8349 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -205,12 +205,6 @@ inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& } } -//import textures that are only supported in pbr contexts -inline void SetMaterialPBRTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, unsigned int texSlot) -{ - return SetMaterialTextureProperty(embeddedTexIdxs, r, prop, mat, aiTextureType_UNKNOWN, texSlot); -} - inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType) { return SetMaterialTextureProperty(embeddedTexIdxs, r, prop, mat, texType, -1); @@ -231,7 +225,7 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) SetMaterialColorProperty(r, mat.baseColorFactor, aimat, AI_MATKEY_COLOR_DIFFUSE); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.baseColorTexture, aimat, aiTextureType_DIFFUSE); - SetMaterialPBRTextureProperty(embeddedTexIdxs, r, mat.metallicRoughnessTexture, aimat, 0); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.metallicRoughnessTexture, aimat, aiTextureType_UNKNOWN, 0); aimat->AddProperty(&mat.metallicFactor, 1, "$mat.gltf.metallicFactor"); aimat->AddProperty(&mat.roughnessFactor, 1, "$mat.gltf.roughnessFactor"); From 863458cd4a30632350685f8aee88ed420118f577 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 31 Aug 2017 01:35:10 -0400 Subject: [PATCH 099/305] Start removing materials common, and adding pbrSpecularGlossiness --- code/glTF2Asset.h | 69 +++++---------------------------------- code/glTF2Asset.inl | 64 ++++++++++++------------------------ code/glTF2AssetWriter.h | 2 +- code/glTF2AssetWriter.inl | 42 +++++++++++++++--------- code/glTF2Importer.cpp | 7 ++++ 5 files changed, 63 insertions(+), 121 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 509fef115..b19de747a 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -44,7 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * glTF Extensions Support: * KHR_binary_glTF: full - * KHR_materials_common: full */ #ifndef GLTF2ASSET_H_INC #define GLTF2ASSET_H_INC @@ -697,19 +696,6 @@ namespace glTF2 //! The material appearance of a primitive. struct Material : public Object { - //Ref source; //!< The ID of the technique. - //std::gltf_unordered_map values; //!< A dictionary object of parameter values. - - //! Techniques defined by KHR_materials_common - enum Technique - { - Technique_undefined = 0, - Technique_BLINN, - Technique_PHONG, - Technique_LAMBERT, - Technique_CONSTANT - }; - //PBR metallic roughness properties vec4 baseColorFactor; TextureInfo baseColorTexture; @@ -726,13 +712,12 @@ namespace glTF2 float alphaCutoff; bool doubleSided; - //fallback material properties (compatible with non-pbr defintions) - /*FallbackTexProperty diffuse; - FallbackTexProperty emission; - FallbackTexProperty specular; - Ref normal;*/ - - Technique technique; + //extension: KHR_materials_pbrSpecularGlossiness + vec4 diffuseFactor; + vec3 specularFactor; + float glossinessFactor; + TextureInfo diffuseTexture; + TextureInfo specularGlossinessTexture; Material() { SetDefaults(); } void Read(Value& obj, Asset& r); @@ -943,35 +928,6 @@ namespace glTF2 void Read(Value& obj, Asset& r); }; - - //! A light (from KHR_materials_common extension) - struct Light : public Object - { - enum Type - { - Type_undefined, - Type_ambient, - Type_directional, - Type_point, - Type_spot - }; - - Type type; - - vec4 color; - float distance; - float constantAttenuation; - float linearAttenuation; - float quadraticAttenuation; - float falloffAngle; - float falloffExponent; - - Light() {} - void Read(Value& obj, Asset& r); - - void SetDefaults(); - }; - struct Animation : public Object { struct AnimSampler { @@ -1152,7 +1108,7 @@ namespace glTF2 struct Extensions { bool KHR_binary_glTF; - bool KHR_materials_common; + bool KHR_materials_pbrSpecularGlossiness; } extensionsUsed; @@ -1170,16 +1126,11 @@ namespace glTF2 LazyDict materials; LazyDict meshes; LazyDict nodes; - //LazyDict programs; LazyDict samplers; LazyDict scenes; - //LazyDict shaders; LazyDict skins; - //LazyDict techniques; LazyDict textures; - LazyDict lights; // KHR_materials_common ext - Ref scene; public: @@ -1195,14 +1146,10 @@ namespace glTF2 , materials (*this, "materials") , meshes (*this, "meshes") , nodes (*this, "nodes") - //, programs (*this, "programs") , samplers (*this, "samplers") , scenes (*this, "scenes") - //, shaders (*this, "shaders") - , skins (*this, "skins") - //, techniques (*this, "techniques") + , skins (*this, "skins") , textures (*this, "textures") - , lights (*this, "lights", "KHR_materials_common") { memset(&extensionsUsed, 0, sizeof(extensionsUsed)); } diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 94858044d..3d53bfccf 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -757,12 +757,12 @@ inline void Material::Read(Value& material, Asset& r) { SetDefaults(); - if (Value* values = FindObject(material, "pbrMetallicRoughness")) { - ReadMember(*values, "baseColorFactor", this->baseColorFactor); - ReadTextureProperty(r, *values, "baseColorTexture", this->baseColorTexture); - ReadTextureProperty(r, *values, "metallicRoughnessTexture", this->metallicRoughnessTexture); - ReadMember(*values, "metallicFactor", this->metallicFactor); - ReadMember(*values, "roughnessFactor", this->roughnessFactor); + if (Value* pbrMetallicRoughness = FindObject(material, "pbrMetallicRoughness")) { + ReadMember(*pbrMetallicRoughness, "baseColorFactor", this->baseColorFactor); + ReadTextureProperty(r, *pbrMetallicRoughness, "baseColorTexture", this->baseColorTexture); + ReadTextureProperty(r, *pbrMetallicRoughness, "metallicRoughnessTexture", this->metallicRoughnessTexture); + ReadMember(*pbrMetallicRoughness, "metallicFactor", this->metallicFactor); + ReadMember(*pbrMetallicRoughness, "roughnessFactor", this->roughnessFactor); } ReadTextureProperty(r, material, "normalTexture", this->normalTexture); @@ -774,30 +774,17 @@ inline void Material::Read(Value& material, Asset& r) ReadMember(material, "alphaMode", this->alphaMode); ReadMember(material, "alphaCutoff", this->alphaCutoff); - /* if (Value* extensions = FindObject(material, "extensions")) { - if (r.extensionsUsed.KHR_materials_common) { - if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { - if (Value* tnq = FindString(*ext, "technique")) { - const char* t = tnq->GetString(); - if (strcmp(t, "BLINN") == 0) technique = Technique_BLINN; - else if (strcmp(t, "PHONG") == 0) technique = Technique_PHONG; - else if (strcmp(t, "LAMBERT") == 0) technique = Technique_LAMBERT; - else if (strcmp(t, "CONSTANT") == 0) technique = Technique_CONSTANT; - } - - if (Value* values = FindObject(*ext, "values")) { - ReadTextureProperty(r, *values, "ambient", this->ambient); - ReadTextureProperty(r, *values, "diffuse", this->diffuse); - ReadTextureProperty(r, *values, "specular", this->specular); - - ReadMember(*values, "doubleSided", doubleSided); - ReadMember(*values, "transparent", transparent); - ReadMember(*values, "transparency", transparency); - ReadMember(*values, "shininess", shininess); - } + if (Value* extensions = FindObject(material, "extensions")) { + if (r.extensionsUsed.KHR_materials_pbrSpecularGlossiness) { + if (Value* pbrSpecularGlossiness = FindObject(*extensions, "KHR_materials_pbrSpecularGlossiness")) { + ReadMember(*pbrSpecularGlossiness, "diffuseFactor", this->diffuseFactor); + ReadTextureProperty(r, *pbrSpecularGlossiness, "diffuseTexture", this->diffuseTexture); + ReadTextureProperty(r, *pbrSpecularGlossiness, "specularGlossinessTexture", this->specularGlossinessTexture); + ReadMember(*pbrSpecularGlossiness, "specularFactor", this->specularFactor); + ReadMember(*pbrSpecularGlossiness, "glossinessFactor", this->glossinessFactor); } } - } */ + } } namespace { @@ -820,7 +807,10 @@ inline void Material::SetDefaults() alphaCutoff = 0.5; doubleSided = false; - technique = Technique_undefined; + //pbrSpecularGlossiness properties + SetVector(diffuseFactor, 1, 1, 1, 1); + SetVector(specularFactor, 1, 1, 1); + glossinessFactor = 1.0; } namespace { @@ -1083,20 +1073,6 @@ inline void Node::Read(Value& obj, Asset& r) if (this->camera) this->camera->id = this->id; } - - // TODO load "skeletons", "skin", "jointName" - - /*if (Value* extensions = FindObject(obj, "extensions")) { - if (r.extensionsUsed.KHR_materials_common) { - - if (Value* ext = FindObject(*extensions, "KHR_materials_common")) { - if (Value* light = FindUInt(*ext, "light")) { - this->light = r.lights.Retrieve(light->GetUint()); - } - } - - } - }*/ } inline void Scene::Read(Value& obj, Asset& r) @@ -1293,7 +1269,7 @@ inline void Asset::ReadExtensionsUsed(Document& doc) if (exts.find(#EXT) != exts.end()) extensionsUsed.EXT = true; CHECK_EXT(KHR_binary_glTF); - CHECK_EXT(KHR_materials_common); + CHECK_EXT(KHR_materials_pbrSpecularGlossiness); #undef CHECK_EXT } diff --git a/code/glTF2AssetWriter.h b/code/glTF2AssetWriter.h index 976f23f9a..97efbeab5 100644 --- a/code/glTF2AssetWriter.h +++ b/code/glTF2AssetWriter.h @@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * glTF Extensions Support: * KHR_binary_glTF: full - * KHR_materials_common: full + * KHR_materials_pbrSpecularGlossiness: full */ #ifndef GLTF2ASSETWRITER_H_INC #define GLTF2ASSETWRITER_H_INC diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index bd16a7023..33625816e 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -327,23 +327,35 @@ namespace glTF2 { obj.AddMember("doubleSided", m.doubleSided, w.mAl); } - /*Value v; - v.SetObject(); + Value pbrSpecularGlossiness; + pbrSpecularGlossiness.SetObject(); { - if (m.transparent && !m.diffuse.texture) { - m.diffuse.color[3] = m.transparency; + //pbrSpecularGlossiness + + vec4 defaultDiffuseFactor = {1, 1, 1, 1}; + WriteVec(pbrSpecularGlossiness, m.diffuseFactor, "diffuseFactor", defaultDiffuseFactor, w.mAl); + + vec3 defaultSpecularFactor = {1, 1, 1}; + WriteVec(pbrSpecularGlossiness, m.specularFactor, "specularFactor", defaultSpecularFactor, w.mAl); + + if (m.glossinessFactor != 1) { + WriteFloat(obj, m.glossinessFactor, "glossinessFactor", w.mAl); } - WriteVecOrTex(v, m.ambient, m.ambient.texture ? "ambientTexture" : "ambientFactor", w.mAl); - WriteVecOrTex(v, m.diffuse, m.diffuse.texture ? "diffuseTexture" : "diffuseFactor", w.mAl); - WriteVecOrTex(v, m.specular, m.specular.texture ? "specularTexture" : "specularFactor", w.mAl); - WriteVecOrTex(v, m.emission, m.emission.texture ? "emissionTexture" : "emissionFactor", w.mAl); - v.AddMember("shininessFactor", m.shininess, w.mAl); + + WriteTex(obj, m.diffuseTexture, "diffuseTexture", w.mAl); + WriteTex(obj, m.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); } - v.AddMember("type", "commonPhong", w.mAl); + Value ext; ext.SetObject(); - ext.AddMember("KHR_materials_common", v, w.mAl); - obj.AddMember("extensions", ext, w.mAl);*/ + + if (!pbrSpecularGlossiness.ObjectEmpty()) { + ext.AddMember("KHR_materials_pbrSpecularGlossiness", pbrSpecularGlossiness, w.mAl); + } + + if (!ext.ObjectEmpty()) { + obj.AddMember("extensions", ext, w.mAl); + } } namespace { @@ -714,11 +726,11 @@ namespace glTF2 { Value exts; exts.SetArray(); { - if (false) - exts.PushBack(StringRef("KHR_binary_glTF"), mAl); + //if (false) + // exts.PushBack(StringRef("KHR_binary_glTF"), mAl); // This is used to export common materials with GLTF 2. - //exts.PushBack(StringRef("KHR_materials_common"), mAl); + //exts.PushBack(StringRef("KHR_materials_pbrSpecularGlossiness"), mAl); } if (!exts.Empty()) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index b1bbf8349..316970fa5 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -237,6 +237,13 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) aimat->AddProperty(&mat.doubleSided, 1, AI_MATKEY_TWOSIDED); aimat->AddProperty(&mat.alphaMode, 1, "$mat.gltf.alphaMode"); aimat->AddProperty(&mat.alphaCutoff, 1, "$mat.gltf.alphaCutoff"); + + //pbrSpecularGlossiness + SetMaterialColorProperty(r, mat.diffuseFactor, aimat, "$clr.diffuse", 0, 1); + SetMaterialColorProperty(r, mat.specularFactor, aimat, "$clr.specular", 0, 1); + aimat->AddProperty(&mat.glossinessFactor, 1, "$mat.gltf.glossinessFactor"); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.diffuseTexture, aimat, aiTextureType_DIFFUSE, 1); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.specularGlossinessTexture, aimat, aiTextureType_UNKNOWN, 1); } } From 7532d6aac1f18aba3b7e33e7f706c143b6430d4d Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 31 Aug 2017 18:26:50 -0400 Subject: [PATCH 100/305] Remove Light, Technique references --- code/glTF2Asset.h | 23 ------------------- code/glTF2Asset.inl | 46 -------------------------------------- code/glTF2AssetWriter.inl | 10 --------- code/glTF2Importer.cpp | 47 ++------------------------------------- 4 files changed, 2 insertions(+), 124 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index b19de747a..a4b833f58 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -129,7 +129,6 @@ namespace glTF2 struct BufferView; // here due to cross-reference struct Texture; - struct Light; struct Skin; // Vec/matrix types, as raw float arrays @@ -835,7 +834,6 @@ namespace glTF2 Nullable scale; Ref camera; - Ref light; std::vector< Ref > skeletons; //!< The ID of skeleton nodes. Each of which is the root of a node hierarchy. Ref skin; //!< The ID of the skin referenced by this node. @@ -891,27 +889,6 @@ namespace glTF2 void Read(Value& obj, Asset& r); }; - struct Technique : public Object - { - struct Parameters - { - - }; - - struct States - { - - }; - - struct Functions - { - - }; - - Technique() {} - void Read(Value& obj, Asset& r); - }; - //! A texture and its sampler. struct Texture : public Object { diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 3d53bfccf..eab0b8373 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -984,52 +984,6 @@ inline void Camera::Read(Value& obj, Asset& r) } } -inline void Light::Read(Value& obj, Asset& r) -{ - SetDefaults(); - - if (Value* type = FindString(obj, "type")) { - const char* t = type->GetString(); - if (strcmp(t, "ambient") == 0) this->type = Type_ambient; - else if (strcmp(t, "directional") == 0) this->type = Type_directional; - else if (strcmp(t, "point") == 0) this->type = Type_point; - else if (strcmp(t, "spot") == 0) this->type = Type_spot; - - if (this->type != Type_undefined) { - if (Value* vals = FindString(obj, t)) { - ReadMember(*vals, "color", color); - - ReadMember(*vals, "constantAttenuation", constantAttenuation); - ReadMember(*vals, "linearAttenuation", linearAttenuation); - ReadMember(*vals, "quadraticAttenuation", quadraticAttenuation); - ReadMember(*vals, "distance", distance); - - ReadMember(*vals, "falloffAngle", falloffAngle); - ReadMember(*vals, "falloffExponent", falloffExponent); - } - } - } -} - -inline void Light::SetDefaults() -{ - #ifndef M_PI - const float M_PI = 3.14159265358979323846f; - #endif - - type = Type_undefined; - - SetVector(color, 0.f, 0.f, 0.f, 1.f); - - constantAttenuation = 0.f; - linearAttenuation = 1.f; - quadraticAttenuation = 1.f; - distance = 0.f; - - falloffAngle = static_cast(M_PI / 2.f); - falloffExponent = 0.f; -} - inline void Node::Read(Value& obj, Asset& r) { diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 33625816e..a8ee16c82 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -557,11 +557,6 @@ namespace glTF2 { } - inline void Write(Value& obj, Technique& b, AssetWriter& w) - { - - } - inline void Write(Value& obj, Texture& tex, AssetWriter& w) { if (tex.source) { @@ -572,11 +567,6 @@ namespace glTF2 { } } - inline void Write(Value& obj, Light& b, AssetWriter& w) - { - - } - inline AssetWriter::AssetWriter(Asset& a) : mDoc() diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 316970fa5..b4fd96948 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -493,46 +493,6 @@ void glTF2Importer::ImportCameras(glTF2::Asset& r) } } -void glTF2Importer::ImportLights(glTF2::Asset& r) -{ - if (!r.lights.Size()) return; - - mScene->mNumLights = r.lights.Size(); - mScene->mLights = new aiLight*[r.lights.Size()]; - - for (size_t i = 0; i < r.lights.Size(); ++i) { - Light& l = r.lights[i]; - - aiLight* ail = mScene->mLights[i] = new aiLight(); - - switch (l.type) { - case Light::Type_directional: - ail->mType = aiLightSource_DIRECTIONAL; break; - - case Light::Type_spot: - ail->mType = aiLightSource_SPOT; break; - - case Light::Type_ambient: - ail->mType = aiLightSource_AMBIENT; break; - - default: // Light::Type_point - ail->mType = aiLightSource_POINT; break; - } - - CopyValue(l.color, ail->mColorAmbient); - CopyValue(l.color, ail->mColorDiffuse); - CopyValue(l.color, ail->mColorSpecular); - - ail->mAngleOuterCone = l.falloffAngle; - ail->mAngleInnerCone = l.falloffExponent; // TODO fix this, it does not look right at all - - ail->mAttenuationConstant = l.constantAttenuation; - ail->mAttenuationLinear = l.linearAttenuation; - ail->mAttenuationQuadratic = l.quadraticAttenuation; - } -} - - aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& meshOffsets, glTF2::Ref& ptr) { Node& node = *ptr; @@ -602,10 +562,6 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& pScene->mCameras[node.camera.GetIndex()]->mName = ainode->mName; } - if (node.light) { - pScene->mLights[node.light.GetIndex()]->mName = ainode->mName; - } - return ainode; } @@ -695,13 +651,14 @@ void glTF2Importer::InternReadFile(const std::string& pFile, aiScene* pScene, IO // Copy the data out // + + ImportEmbeddedTextures(asset); ImportMaterials(asset); ImportMeshes(asset); ImportCameras(asset); - ImportLights(asset); ImportNodes(asset); From 562920fbb8811bd8b47eb9eb5755ad25c0b98973 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 31 Aug 2017 18:30:43 -0400 Subject: [PATCH 101/305] Changes to GLTF2 materials pbrMetallicRoughness and pbrSpecularGlossiness as structs; persist textureinfo properties from start to finish; persist pbrSpecularGlossiness (via extensionsUsed) usage from start to finish --- code/glTF2Asset.h | 27 +++++++--- code/glTF2Asset.inl | 36 +++++++------ code/glTF2AssetWriter.inl | 64 +++++++++++----------- code/glTF2Exporter.cpp | 109 +++++++++++++++++++++++++++++++++----- code/glTF2Exporter.h | 8 +++ code/glTF2Importer.cpp | 48 +++++++++-------- 6 files changed, 200 insertions(+), 92 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index a4b833f58..3faea0938 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -692,15 +692,30 @@ namespace glTF2 vec4 color; };*/ - //! The material appearance of a primitive. - struct Material : public Object + struct PbrMetallicRoughness { - //PBR metallic roughness properties vec4 baseColorFactor; TextureInfo baseColorTexture; TextureInfo metallicRoughnessTexture; float metallicFactor; float roughnessFactor; + }; + + struct PbrSpecularGlossiness + { + bool on = false; + vec4 diffuseFactor; + vec3 specularFactor; + float glossinessFactor; + TextureInfo diffuseTexture; + TextureInfo specularGlossinessTexture; + }; + + //! The material appearance of a primitive. + struct Material : public Object + { + //PBR metallic roughness properties + PbrMetallicRoughness pbrMetallicRoughness; //other basic material properties NormalTextureInfo normalTexture; @@ -712,11 +727,7 @@ namespace glTF2 bool doubleSided; //extension: KHR_materials_pbrSpecularGlossiness - vec4 diffuseFactor; - vec3 specularFactor; - float glossinessFactor; - TextureInfo diffuseTexture; - TextureInfo specularGlossinessTexture; + PbrSpecularGlossiness pbrSpecularGlossiness; Material() { SetDefaults(); } void Read(Value& obj, Asset& r); diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index eab0b8373..632401576 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -758,11 +758,11 @@ inline void Material::Read(Value& material, Asset& r) SetDefaults(); if (Value* pbrMetallicRoughness = FindObject(material, "pbrMetallicRoughness")) { - ReadMember(*pbrMetallicRoughness, "baseColorFactor", this->baseColorFactor); - ReadTextureProperty(r, *pbrMetallicRoughness, "baseColorTexture", this->baseColorTexture); - ReadTextureProperty(r, *pbrMetallicRoughness, "metallicRoughnessTexture", this->metallicRoughnessTexture); - ReadMember(*pbrMetallicRoughness, "metallicFactor", this->metallicFactor); - ReadMember(*pbrMetallicRoughness, "roughnessFactor", this->roughnessFactor); + ReadMember(*pbrMetallicRoughness, "baseColorFactor", this->pbrMetallicRoughness.baseColorFactor); + ReadTextureProperty(r, *pbrMetallicRoughness, "baseColorTexture", this->pbrMetallicRoughness.baseColorTexture); + ReadTextureProperty(r, *pbrMetallicRoughness, "metallicRoughnessTexture", this->pbrMetallicRoughness.metallicRoughnessTexture); + ReadMember(*pbrMetallicRoughness, "metallicFactor", this->pbrMetallicRoughness.metallicFactor); + ReadMember(*pbrMetallicRoughness, "roughnessFactor", this->pbrMetallicRoughness.roughnessFactor); } ReadTextureProperty(r, material, "normalTexture", this->normalTexture); @@ -777,11 +777,13 @@ inline void Material::Read(Value& material, Asset& r) if (Value* extensions = FindObject(material, "extensions")) { if (r.extensionsUsed.KHR_materials_pbrSpecularGlossiness) { if (Value* pbrSpecularGlossiness = FindObject(*extensions, "KHR_materials_pbrSpecularGlossiness")) { - ReadMember(*pbrSpecularGlossiness, "diffuseFactor", this->diffuseFactor); - ReadTextureProperty(r, *pbrSpecularGlossiness, "diffuseTexture", this->diffuseTexture); - ReadTextureProperty(r, *pbrSpecularGlossiness, "specularGlossinessTexture", this->specularGlossinessTexture); - ReadMember(*pbrSpecularGlossiness, "specularFactor", this->specularFactor); - ReadMember(*pbrSpecularGlossiness, "glossinessFactor", this->glossinessFactor); + this->pbrSpecularGlossiness.on = true; + + ReadMember(*pbrSpecularGlossiness, "diffuseFactor", this->pbrSpecularGlossiness.diffuseFactor); + ReadTextureProperty(r, *pbrSpecularGlossiness, "diffuseTexture", this->pbrSpecularGlossiness.diffuseTexture); + ReadTextureProperty(r, *pbrSpecularGlossiness, "specularGlossinessTexture", this->pbrSpecularGlossiness.specularGlossinessTexture); + ReadMember(*pbrSpecularGlossiness, "specularFactor", this->pbrSpecularGlossiness.specularFactor); + ReadMember(*pbrSpecularGlossiness, "glossinessFactor", this->pbrSpecularGlossiness.glossinessFactor); } } } @@ -798,19 +800,19 @@ namespace { inline void Material::SetDefaults() { //pbr materials - SetVector(baseColorFactor, 1, 1, 1, 1); - SetVector(emissiveFactor, 0, 0, 0); - metallicFactor = 1.0; - roughnessFactor = 1.0; + SetVector(pbrMetallicRoughness.baseColorFactor, 1, 1, 1, 1); + pbrMetallicRoughness.metallicFactor = 1.0; + pbrMetallicRoughness.roughnessFactor = 1.0; + SetVector(emissiveFactor, 0, 0, 0); alphaMode = "OPAQUE"; alphaCutoff = 0.5; doubleSided = false; //pbrSpecularGlossiness properties - SetVector(diffuseFactor, 1, 1, 1, 1); - SetVector(specularFactor, 1, 1, 1); - glossinessFactor = 1.0; + SetVector(pbrSpecularGlossiness.diffuseFactor, 1, 1, 1, 1); + SetVector(pbrSpecularGlossiness.specularFactor, 1, 1, 1); + pbrSpecularGlossiness.glossinessFactor = 1.0; } namespace { diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index a8ee16c82..ce2e7c41d 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -287,19 +287,19 @@ namespace glTF2 { Value pbrMetallicRoughness; pbrMetallicRoughness.SetObject(); { - WriteTex(pbrMetallicRoughness, m.baseColorTexture, "baseColorTexture", w.mAl); - WriteTex(pbrMetallicRoughness, m.metallicRoughnessTexture, "metallicRoughnessTexture", w.mAl); + WriteTex(pbrMetallicRoughness, m.pbrMetallicRoughness.baseColorTexture, "baseColorTexture", w.mAl); + WriteTex(pbrMetallicRoughness, m.pbrMetallicRoughness.metallicRoughnessTexture, "metallicRoughnessTexture", w.mAl); //@TODO: define this as a constant? vec4 defaultEmissiveFactor = {1, 1, 1, 1}; - WriteVec(pbrMetallicRoughness, m.baseColorFactor, "baseColorFactor", defaultEmissiveFactor, w.mAl); + WriteVec(pbrMetallicRoughness, m.pbrMetallicRoughness.baseColorFactor, "baseColorFactor", defaultEmissiveFactor, w.mAl); - if (m.metallicFactor != 1) { - WriteFloat(pbrMetallicRoughness, m.metallicFactor, "metallicFactor", w.mAl); + if (m.pbrMetallicRoughness.metallicFactor != 1) { + WriteFloat(pbrMetallicRoughness, m.pbrMetallicRoughness.metallicFactor, "metallicFactor", w.mAl); } - if (m.roughnessFactor != 1) { - WriteFloat(pbrMetallicRoughness, m.roughnessFactor, "roughnessFactor", w.mAl); + if (m.pbrMetallicRoughness.roughnessFactor != 1) { + WriteFloat(pbrMetallicRoughness, m.pbrMetallicRoughness.roughnessFactor, "roughnessFactor", w.mAl); } } @@ -327,34 +327,36 @@ namespace glTF2 { obj.AddMember("doubleSided", m.doubleSided, w.mAl); } - Value pbrSpecularGlossiness; - pbrSpecularGlossiness.SetObject(); - { - //pbrSpecularGlossiness + Value exts; + exts.SetObject(); - vec4 defaultDiffuseFactor = {1, 1, 1, 1}; - WriteVec(pbrSpecularGlossiness, m.diffuseFactor, "diffuseFactor", defaultDiffuseFactor, w.mAl); + if (m.pbrSpecularGlossiness.on) { + Value pbrSpecularGlossiness; + pbrSpecularGlossiness.SetObject(); + { + //pbrSpecularGlossiness - vec3 defaultSpecularFactor = {1, 1, 1}; - WriteVec(pbrSpecularGlossiness, m.specularFactor, "specularFactor", defaultSpecularFactor, w.mAl); + vec4 defaultDiffuseFactor = {1, 1, 1, 1}; + WriteVec(pbrSpecularGlossiness, m.pbrSpecularGlossiness.diffuseFactor, "diffuseFactor", defaultDiffuseFactor, w.mAl); - if (m.glossinessFactor != 1) { - WriteFloat(obj, m.glossinessFactor, "glossinessFactor", w.mAl); + vec3 defaultSpecularFactor = {1, 1, 1}; + WriteVec(pbrSpecularGlossiness, m.pbrSpecularGlossiness.specularFactor, "specularFactor", defaultSpecularFactor, w.mAl); + + if (m.pbrSpecularGlossiness.glossinessFactor != 1) { + WriteFloat(obj, m.pbrSpecularGlossiness.glossinessFactor, "glossinessFactor", w.mAl); + } + + WriteTex(obj, m.pbrSpecularGlossiness.diffuseTexture, "diffuseTexture", w.mAl); + WriteTex(obj, m.pbrSpecularGlossiness.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); } - WriteTex(obj, m.diffuseTexture, "diffuseTexture", w.mAl); - WriteTex(obj, m.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); + if (!pbrSpecularGlossiness.ObjectEmpty()) { + exts.AddMember("KHR_materials_pbrSpecularGlossiness", pbrSpecularGlossiness, w.mAl); + } } - Value ext; - ext.SetObject(); - - if (!pbrSpecularGlossiness.ObjectEmpty()) { - ext.AddMember("KHR_materials_pbrSpecularGlossiness", pbrSpecularGlossiness, w.mAl); - } - - if (!ext.ObjectEmpty()) { - obj.AddMember("extensions", ext, w.mAl); + if (!exts.ObjectEmpty()) { + obj.AddMember("extensions", exts, w.mAl); } } @@ -719,8 +721,10 @@ namespace glTF2 { //if (false) // exts.PushBack(StringRef("KHR_binary_glTF"), mAl); - // This is used to export common materials with GLTF 2. - //exts.PushBack(StringRef("KHR_materials_pbrSpecularGlossiness"), mAl); + // This is used to export pbrSpecularGlossiness materials with GLTF 2. + if (this->mAsset.extensionsUsed.KHR_materials_pbrSpecularGlossiness) { + exts.PushBack(StringRef("KHR_materials_pbrSpecularGlossiness"), mAl); + } } if (!exts.Empty()) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index f59c280f1..9e04da76c 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -112,9 +112,9 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai ExportMetadata(); - //for (unsigned int i = 0; i < pScene->mNumCameras; ++i) {} - - //for (unsigned int i = 0; i < pScene->mNumLights; ++i) {} + if (mScene->mRootNode) { + ExportExtensions(mScene->mRootNode); + } ExportMaterials(); @@ -124,8 +124,6 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai ExportMeshes(); - //for (unsigned int i = 0; i < pScene->mNumTextures; ++i) {} - ExportScene(); //ExportAnimations(); @@ -280,10 +278,26 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture) texture->sampler->minFilter = SamplerMinFilter_Linear_Mipmap_Linear; } +void glTF2Exporter::GetMatTexProp(const aiMaterial* mat, unsigned int& prop, const char* propName, aiTextureType tt, unsigned int slot) +{ + const char* key = (std::string(_AI_MATKEY_TEXTURE_BASE) + "." + propName).c_str(); + + mat->Get(key, tt, slot, prop); +} + +void glTF2Exporter::GetMatTexProp(const aiMaterial* mat, float& prop, const char* propName, aiTextureType tt, unsigned int slot) +{ + const char* key = (std::string(_AI_MATKEY_TEXTURE_BASE) + "." + propName).c_str(); + + mat->Get(key, tt, slot, prop); +} + void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt, unsigned int slot = 0) { - aiString tex; + if (mat->GetTextureCount(tt) > 0) { + aiString tex; + if (mat->Get(AI_MATKEY_TEXTURE(tt, slot), tex) == AI_SUCCESS) { std::string path = tex.C_Str(); @@ -326,6 +340,41 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe } } +void glTF2Exporter::GetMatTex(const aiMaterial* mat, TextureInfo& prop, aiTextureType tt, unsigned int slot = 0) +{ + Ref& texture = prop.texture; + + GetMatTex(mat, texture, tt, slot); + + if (texture) { + GetMatTexProp(mat, prop.texCoord, "texCoord", tt, slot); + } +} + +void glTF2Exporter::GetMatTex(const aiMaterial* mat, NormalTextureInfo& prop, aiTextureType tt, unsigned int slot = 0) +{ + Ref& texture = prop.texture; + + GetMatTex(mat, texture, tt, slot); + + if (texture) { + GetMatTexProp(mat, prop.texCoord, "texCoord", tt, slot); + GetMatTexProp(mat, prop.scale, "scale", tt, slot); + } +} + +void glTF2Exporter::GetMatTex(const aiMaterial* mat, OcclusionTextureInfo& prop, aiTextureType tt, unsigned int slot = 0) +{ + Ref& texture = prop.texture; + + GetMatTex(mat, texture, tt, slot); + + if (texture) { + GetMatTexProp(mat, prop.texCoord, "texCoord", tt, slot); + GetMatTexProp(mat, prop.strength, "strength", tt, slot); + } +} + void glTF2Exporter::GetMatColor(const aiMaterial* mat, vec4& prop, const char* propName, int type, int idx) { aiColor4D col; @@ -344,6 +393,8 @@ void glTF2Exporter::GetMatColor(const aiMaterial* mat, vec3& prop, const char* p void glTF2Exporter::ExportMaterials() { + bool& KHR_materials_pbrSpecularGlossiness = mAsset->extensionsUsed.KHR_materials_pbrSpecularGlossiness; + aiString aiName; for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { const aiMaterial* mat = mScene->mMaterials[i]; @@ -356,20 +407,36 @@ void glTF2Exporter::ExportMaterials() Ref m = mAsset->materials.Create(name); - GetMatTex(mat, m->baseColorTexture.texture, aiTextureType_DIFFUSE); - GetMatTex(mat, m->metallicRoughnessTexture.texture, aiTextureType_UNKNOWN, 0);//get unknown slot - GetMatTex(mat, m->emissiveTexture.texture, aiTextureType_EMISSIVE); - GetMatTex(mat, m->normalTexture.texture, aiTextureType_NORMALS); - GetMatTex(mat, m->occlusionTexture.texture, aiTextureType_LIGHTMAP); + GetMatTex(mat, m->pbrMetallicRoughness.baseColorTexture, aiTextureType_DIFFUSE); + GetMatTex(mat, m->pbrMetallicRoughness.metallicRoughnessTexture, aiTextureType_UNKNOWN, 0);//get unknown slot + GetMatColor(mat, m->pbrMetallicRoughness.baseColorFactor, AI_MATKEY_COLOR_DIFFUSE); - GetMatColor(mat, m->baseColorFactor, AI_MATKEY_COLOR_DIFFUSE); + GetMatTex(mat, m->normalTexture, aiTextureType_NORMALS); + GetMatTex(mat, m->occlusionTexture, aiTextureType_LIGHTMAP); + GetMatTex(mat, m->emissiveTexture, aiTextureType_EMISSIVE); GetMatColor(mat, m->emissiveFactor, AI_MATKEY_COLOR_EMISSIVE); mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); mat->Get("$mat.gltf.alphaCutoff", 0, 0, m->alphaCutoff); - mat->Get("$mat.gltf.metallicFactor", 0, 0, m->metallicFactor); - mat->Get("$mat.gltf.roughnessFactor", 0, 0, m->roughnessFactor); + mat->Get("$mat.gltf.metallicFactor", 0, 0, m->pbrMetallicRoughness.metallicFactor); + mat->Get("$mat.gltf.roughnessFactor", 0, 0, m->pbrMetallicRoughness.roughnessFactor); mat->Get("$mat.gltf.alphaMode", 0, 0, m->alphaMode); + + bool hasPbrSpecularGlossiness; + mat->Get("$mat.gltf.pbrSpecularGlossiness.on", 0, 0, hasPbrSpecularGlossiness); + + if (hasPbrSpecularGlossiness) { + + if (!KHR_materials_pbrSpecularGlossiness) { + KHR_materials_pbrSpecularGlossiness = true; + } + + GetMatColor(mat, m->pbrSpecularGlossiness.diffuseFactor, "$clr.diffuse", 0, 1); + GetMatColor(mat, m->pbrSpecularGlossiness.specularFactor, "$clr.specular", 0, 1); + mat->Get("$mat.gltf.glossinessFactor", 0, 0, m->pbrSpecularGlossiness.glossinessFactor); + GetMatTex(mat, m->pbrSpecularGlossiness.diffuseTexture, aiTextureType_DIFFUSE, 1); + GetMatTex(mat, m->pbrSpecularGlossiness.specularGlossinessTexture, aiTextureType_UNKNOWN, 1); + } } } @@ -862,6 +929,20 @@ void glTF2Exporter::ExportMetadata() asset.generator = buffer; } +void glTF2Exporter::ExportExtensions(const aiNode* n) +{ + aiMetadata* mMetaData = n->mMetaData; + + if (mMetaData != nullptr) { + bool pbrSpecularGlossiness; + + if (mMetaData->Get("extensionsUsed.pbrSpecularGlossiness", pbrSpecularGlossiness)) { + mAsset->extensionsUsed.KHR_materials_pbrSpecularGlossiness = pbrSpecularGlossiness; + } + } + +} + inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref& animRef, Ref& buffer, const aiNodeAnim* nodeChannel, float ticksPerSecond) { // Loop over the data and check to see if it exactly matches an existing buffer. diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index 2ba0143bf..827189f29 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -67,6 +67,8 @@ namespace glTF2 class Asset; struct TexProperty; struct TextureInfo; + struct NormalTextureInfo; + struct OcclusionTextureInfo; struct Node; struct Texture; @@ -107,10 +109,16 @@ namespace Assimp void WriteBinaryData(IOStream* outfile, std::size_t sceneLength); void GetTexSampler(const aiMaterial* mat, glTF2::Ref texture); + void GetMatTexProp(const aiMaterial* mat, unsigned int& prop, const char* propName, aiTextureType tt, unsigned int idx); + void GetMatTexProp(const aiMaterial* mat, float& prop, const char* propName, aiTextureType tt, unsigned int idx); void GetMatTex(const aiMaterial* mat, glTF2::Ref& texture, aiTextureType tt, unsigned int slot); + void GetMatTex(const aiMaterial* mat, glTF2::TextureInfo& prop, aiTextureType tt, unsigned int slot); + void GetMatTex(const aiMaterial* mat, glTF2::NormalTextureInfo& prop, aiTextureType tt, unsigned int slot); + void GetMatTex(const aiMaterial* mat, glTF2::OcclusionTextureInfo& prop, aiTextureType tt, unsigned int slot); void GetMatColor(const aiMaterial* mat, glTF2::vec4& prop, const char* propName, int type, int idx); void GetMatColor(const aiMaterial* mat, glTF2::vec3& prop, const char* propName, int type, int idx); void ExportMetadata(); + void ExportExtensions(const aiNode* n); void ExportMaterials(); void ExportMeshes(); unsigned int ExportNodeHierarchy(const aiNode* n); diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index b4fd96948..97901fc01 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -183,7 +183,7 @@ inline void SetMaterialColorProperty(Asset& r, vec3& prop, aiMaterial* mat, cons return SetMaterialColorProperty(r, prop4, mat, pKey, type, idx); } -inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType, int texSlot) +inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType, unsigned int texSlot = 0) { if (prop.texture && prop.texture->source) { aiString uri(prop.texture->source->uri); @@ -195,19 +195,11 @@ inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& uri.length = 1 + ASSIMP_itoa10(uri.data + 1, MAXLEN - 1, texIdx); } - if (texSlot < 0) { - mat->AddProperty(&uri, _AI_MATKEY_TEXTURE_BASE, texType, 0); - } - else { - mat->AddProperty(&uri, AI_MATKEY_TEXTURE(texType, - texSlot)); - } - } -} + mat->AddProperty(&uri, AI_MATKEY_TEXTURE(texType, texSlot)); -inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType) -{ - return SetMaterialTextureProperty(embeddedTexIdxs, r, prop, mat, texType, -1); + const char *texCoordName = (std::string(_AI_MATKEY_TEXTURE_BASE) + ".texCoord").c_str(); + mat->AddProperty(&prop.texCoord, 1, texCoordName, texType, texSlot); + } } void glTF2Importer::ImportMaterials(glTF2::Asset& r) @@ -223,11 +215,11 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) aiString str(mat.id); aimat->AddProperty(&str, AI_MATKEY_NAME); - SetMaterialColorProperty(r, mat.baseColorFactor, aimat, AI_MATKEY_COLOR_DIFFUSE); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.baseColorTexture, aimat, aiTextureType_DIFFUSE); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.metallicRoughnessTexture, aimat, aiTextureType_UNKNOWN, 0); - aimat->AddProperty(&mat.metallicFactor, 1, "$mat.gltf.metallicFactor"); - aimat->AddProperty(&mat.roughnessFactor, 1, "$mat.gltf.roughnessFactor"); + SetMaterialColorProperty(r, mat.pbrMetallicRoughness.baseColorFactor, aimat, AI_MATKEY_COLOR_DIFFUSE); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrMetallicRoughness.baseColorTexture, aimat, aiTextureType_DIFFUSE); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrMetallicRoughness.metallicRoughnessTexture, aimat, aiTextureType_UNKNOWN); + aimat->AddProperty(&mat.pbrMetallicRoughness.metallicFactor, 1, "$mat.gltf.pbrMetallicRoughness.metallicFactor"); + aimat->AddProperty(&mat.pbrMetallicRoughness.roughnessFactor, 1, "$mat.gltf.pbrMetallicRoughness.roughnessFactor"); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.normalTexture, aimat, aiTextureType_NORMALS); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.occlusionTexture, aimat, aiTextureType_LIGHTMAP); @@ -239,11 +231,14 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) aimat->AddProperty(&mat.alphaCutoff, 1, "$mat.gltf.alphaCutoff"); //pbrSpecularGlossiness - SetMaterialColorProperty(r, mat.diffuseFactor, aimat, "$clr.diffuse", 0, 1); - SetMaterialColorProperty(r, mat.specularFactor, aimat, "$clr.specular", 0, 1); - aimat->AddProperty(&mat.glossinessFactor, 1, "$mat.gltf.glossinessFactor"); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.diffuseTexture, aimat, aiTextureType_DIFFUSE, 1); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.specularGlossinessTexture, aimat, aiTextureType_UNKNOWN, 1); + if (mat.pbrSpecularGlossiness.on) { + aimat->AddProperty(&mat.pbrSpecularGlossiness.on, 1, "$mat.gltf.pbrSpecularGlossiness.on"); + SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.diffuseFactor, aimat, "$clr.diffuse", 0, 1); + SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.specularFactor, aimat, "$clr.specular", 0, 1); + aimat->AddProperty(&mat.pbrSpecularGlossiness.glossinessFactor, 1, "$mat.gltf.pbrSpecularGlossiness.glossinessFactor"); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.diffuseTexture, aimat, aiTextureType_DIFFUSE, 1); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.specularGlossinessTexture, aimat, aiTextureType_UNKNOWN, 1); + } } } @@ -590,6 +585,13 @@ void glTF2Importer::ImportNodes(glTF2::Asset& r) //if (!mScene->mRootNode) { // mScene->mRootNode = new aiNode("EMPTY"); //} + + //initialize mMetaData; + aiMetadata* mMetaData = new aiMetadata(); + + //store used glTF extensions on the root node, for a lack of a better place. + mMetaData->Add("extensionsUsed.pbrSpecularGlossiness", r.extensionsUsed.KHR_materials_pbrSpecularGlossiness); + mScene->mRootNode->mMetaData = mMetaData; } void glTF2Importer::ImportEmbeddedTextures(glTF2::Asset& r) From 2d54019b8fb10e46f2b1d4e98e3f1a904de15309 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 31 Aug 2017 23:38:10 -0400 Subject: [PATCH 102/305] Remove OPEN3DGC and compression references --- code/glTF2Asset.h | 71 ------------------- code/glTF2Asset.inl | 74 -------------------- code/glTF2AssetWriter.inl | 51 -------------- code/glTF2Exporter.cpp | 142 +------------------------------------- code/glTF2Importer.cpp | 32 --------- 5 files changed, 1 insertion(+), 369 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 3faea0938..dd348de1a 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -752,86 +752,15 @@ namespace glTF2 Ref material; }; - /// \struct SExtension - /// Extension used for mesh. - struct SExtension - { - /// \enum EType - /// Type of extension. - enum EType - { - #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - Compression_Open3DGC,///< Compression of mesh data using Open3DGC algorithm. - #endif - - Unknown - }; - - EType Type;///< Type of extension. - - /// \fn SExtension - /// Constructor. - /// \param [in] pType - type of extension. - SExtension(const EType pType) - : Type(pType) - {} - - virtual ~SExtension() { - // empty - } - }; - - #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - /// \struct SCompression_Open3DGC - /// Compression of mesh data using Open3DGC algorithm. - struct SCompression_Open3DGC : public SExtension - { - using SExtension::Type; - - std::string Buffer;///< ID of "buffer" used for storing compressed data. - size_t Offset;///< Offset in "bufferView" where compressed data are stored. - size_t Count;///< Count of elements in compressed data. Is always equivalent to size in bytes: look comments for "Type" and "Component_Type". - bool Binary;///< If true then "binary" mode is used for coding, if false - "ascii" mode. - size_t IndicesCount;///< Count of indices in mesh. - size_t VerticesCount;///< Count of vertices in mesh. - // AttribType::Value Type;///< Is always "SCALAR". - // ComponentType Component_Type;///< Is always "ComponentType_UNSIGNED_BYTE" (5121). - - /// \fn SCompression_Open3DGC - /// Constructor. - SCompression_Open3DGC() - : SExtension(Compression_Open3DGC) { - // empty - } - - virtual ~SCompression_Open3DGC() { - // empty - } - }; - #endif - std::vector primitives; - std::list Extension;///< List of extensions used in mesh. Mesh() {} - /// \fn ~Mesh() - /// Destructor. - ~Mesh() { for(std::list::iterator it = Extension.begin(), it_end = Extension.end(); it != it_end; it++) { delete *it; }; } - /// \fn void Read(Value& pJSON_Object, Asset& pAsset_Root) /// Get mesh data from JSON-object and place them to root asset. /// \param [in] pJSON_Object - reference to pJSON-object from which data are read. /// \param [out] pAsset_Root - reference to root assed where data will be stored. void Read(Value& pJSON_Object, Asset& pAsset_Root); - - #ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - /// \fn void Decode_O3DGC(const SCompression_Open3DGC& pCompression_Open3DGC, Asset& pAsset_Root) - /// Decode part of "buffer" which encoded with Open3DGC algorithm. - /// \param [in] pCompression_Open3DGC - reference to structure which describe encoded region. - /// \param [out] pAsset_Root - reference to root assed where data will be stored. - void Decode_O3DGC(const SCompression_Open3DGC& pCompression_Open3DGC, Asset& pAsset_Root); - #endif }; struct Node : public Object diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 632401576..dd110ebb7 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -44,11 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Header files, Assimp #include -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - // Header files, Open3DGC. -# include -#endif - using namespace Assimp; namespace glTF2 { @@ -892,75 +887,6 @@ inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) } } } - - /****************** Mesh extensions ******************/ - Value* json_extensions = FindObject(pJSON_Object, "extensions"); - - if(json_extensions == nullptr) goto mr_skip_extensions; - - for(Value::MemberIterator it_memb = json_extensions->MemberBegin(); it_memb != json_extensions->MemberEnd(); it_memb++) - { -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - if(it_memb->name.GetString() == std::string("Open3DGC-compression")) - { - // Search for compressed data. - // Compressed data contain description of part of "buffer" which is encoded. This part must be decoded and - // new data will replace old encoded part by request. In fact \"compressedData\" is kind of "accessor" structure. - Value* comp_data = FindObject(it_memb->value, "compressedData"); - - if(comp_data == nullptr) throw DeadlyImportError("GLTF: \"Open3DGC-compression\" must has \"compressedData\"."); - - DefaultLogger::get()->info("GLTF: Decompressing Open3DGC data."); - - /************** Read data from JSON-document **************/ - #define MESH_READ_COMPRESSEDDATA_MEMBER(pFieldName, pOut) \ - if(!ReadMember(*comp_data, pFieldName, pOut)) \ - { \ - throw DeadlyImportError(std::string("GLTF: \"compressedData\" must has \"") + pFieldName + "\"."); \ - } - - const char* mode_str; - const char* type_str; - ComponentType component_type; - SCompression_Open3DGC* ext_o3dgc = new SCompression_Open3DGC; - - MESH_READ_COMPRESSEDDATA_MEMBER("buffer", ext_o3dgc->Buffer); - MESH_READ_COMPRESSEDDATA_MEMBER("byteOffset", ext_o3dgc->Offset); - MESH_READ_COMPRESSEDDATA_MEMBER("componentType", component_type); - MESH_READ_COMPRESSEDDATA_MEMBER("type", type_str); - MESH_READ_COMPRESSEDDATA_MEMBER("count", ext_o3dgc->Count); - MESH_READ_COMPRESSEDDATA_MEMBER("mode", mode_str); - MESH_READ_COMPRESSEDDATA_MEMBER("indicesCount", ext_o3dgc->IndicesCount); - MESH_READ_COMPRESSEDDATA_MEMBER("verticesCount", ext_o3dgc->VerticesCount); - - #undef MESH_READ_COMPRESSEDDATA_MEMBER - - // Check some values - if(strcmp(type_str, "SCALAR")) throw DeadlyImportError("GLTF: only \"SCALAR\" type is supported for compressed data."); - if(component_type != ComponentType_UNSIGNED_BYTE) throw DeadlyImportError("GLTF: only \"UNSIGNED_BYTE\" component type is supported for compressed data."); - - // Set read/write data mode. - if(strcmp(mode_str, "binary") == 0) - ext_o3dgc->Binary = true; - else if(strcmp(mode_str, "ascii") == 0) - ext_o3dgc->Binary = false; - else - throw DeadlyImportError(std::string("GLTF: for compressed data supported modes is: \"ascii\", \"binary\". Not the: \"") + mode_str + "\"."); - - /************************ Decoding ************************/ - Decode_O3DGC(*ext_o3dgc, pAsset_Root); - Extension.push_back(ext_o3dgc);// store info in mesh extensions list. - }// if(it_memb->name.GetString() == "Open3DGC-compression") - else -#endif - { - throw DeadlyImportError(std::string("GLTF: Unknown mesh extension: \"") + it_memb->name.GetString() + "\"."); - } - }// for(Value::MemberIterator it_memb = json_extensions->MemberBegin(); it_memb != json_extensions->MemberEnd(); json_extensions++) - -mr_skip_extensions: - - return;// After label some operators must be present. } inline void Camera::Read(Value& obj, Asset& r) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index ce2e7c41d..c74ea4031 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -380,57 +380,6 @@ namespace glTF2 { inline void Write(Value& obj, Mesh& m, AssetWriter& w) { - - /**************** Mesh extensions ****************/ - if(m.Extension.size() > 0) - { - Value json_extensions; - - json_extensions.SetObject(); - for(Mesh::SExtension* ptr_ext : m.Extension) - { - switch(ptr_ext->Type) - { -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - case Mesh::SExtension::EType::Compression_Open3DGC: - { - Value json_comp_data; - Mesh::SCompression_Open3DGC* ptr_ext_comp = (Mesh::SCompression_Open3DGC*)ptr_ext; - - // filling object "compressedData" - json_comp_data.SetObject(); - json_comp_data.AddMember("buffer", ptr_ext_comp->Buffer, w.mAl); - json_comp_data.AddMember("byteOffset", ptr_ext_comp->Offset, w.mAl); - json_comp_data.AddMember("componentType", 5121, w.mAl); - json_comp_data.AddMember("type", "SCALAR", w.mAl); - json_comp_data.AddMember("count", ptr_ext_comp->Count, w.mAl); - if(ptr_ext_comp->Binary) - json_comp_data.AddMember("mode", "binary", w.mAl); - else - json_comp_data.AddMember("mode", "ascii", w.mAl); - - json_comp_data.AddMember("indicesCount", ptr_ext_comp->IndicesCount, w.mAl); - json_comp_data.AddMember("verticesCount", ptr_ext_comp->VerticesCount, w.mAl); - // filling object "Open3DGC-compression" - Value json_o3dgc; - - json_o3dgc.SetObject(); - json_o3dgc.AddMember("compressedData", json_comp_data, w.mAl); - // add member to object "extensions" - json_extensions.AddMember("Open3DGC-compression", json_o3dgc, w.mAl); - } - - break; -#endif - default: - throw DeadlyImportError("GLTF: Can not write mesh: unknown mesh extension, only Open3DGC is supported."); - }// switch(ptr_ext->Type) - }// for(Mesh::SExtension* ptr_ext : m.Extension) - - // Add extensions to mesh - obj.AddMember("extensions", json_extensions, w.mAl); - }// if(m.Extension.size() > 0) - /****************** Primitives *******************/ Value primitives; primitives.SetArray(); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 9e04da76c..2b7e4c33d 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -62,11 +62,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "glTF2AssetWriter.h" -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - // Header files, Open3DGC. -# include -#endif - using namespace rapidjson; using namespace Assimp; @@ -573,15 +568,6 @@ void glTF2Exporter::ExportMeshes() // because "ComponentType_UNSIGNED_SHORT" used for indices. And it's a maximal type according to glTF specification. typedef unsigned short IndicesType; - // Variables needed for compression. BEGIN. - // Indices, not pointers - because pointer to buffer is changing while writing to it. - size_t idx_srcdata_begin;// Index of buffer before writing mesh data. Also, index of begin of coordinates array in buffer. - size_t idx_srcdata_normal = SIZE_MAX;// Index of begin of normals array in buffer. SIZE_MAX - mean that mesh has no normals. - std::vector idx_srcdata_tc;// Array of indices. Every index point to begin of texture coordinates array in buffer. - size_t idx_srcdata_ind;// Index of begin of coordinates indices array in buffer. - bool comp_allow;// Point that data of current mesh can be compressed. - // Variables needed for compression. END. - std::string fname = std::string(mFilename); std::string bufferIdPrefix = fname.substr(0, fname.rfind(".gltf")); std::string bufferId = mAsset->FindUniqueID("", bufferIdPrefix.c_str()); @@ -614,31 +600,6 @@ void glTF2Exporter::ExportMeshes() for (unsigned int idx_mesh = 0; idx_mesh < mScene->mNumMeshes; ++idx_mesh) { const aiMesh* aim = mScene->mMeshes[idx_mesh]; - // Check if compressing requested and mesh can be encoded. -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - comp_allow = mProperties->GetPropertyBool("extensions.Open3DGC.use", false); -#else - comp_allow = false; -#endif - - if(comp_allow && (aim->mPrimitiveTypes == aiPrimitiveType_TRIANGLE) && (aim->mNumVertices > 0) && (aim->mNumFaces > 0)) - { - idx_srcdata_tc.clear(); - idx_srcdata_tc.reserve(AI_MAX_NUMBER_OF_TEXTURECOORDS); - } - else - { - std::string msg; - - if(aim->mPrimitiveTypes != aiPrimitiveType_TRIANGLE) - msg = "all primitives of the mesh must be a triangles."; - else - msg = "mesh must has vertices and faces."; - - DefaultLogger::get()->warn("GLTF: can not use Open3DGC-compression: " + msg); - comp_allow = false; - } - std::string name = aim->mName.C_Str(); std::string meshId = mAsset->FindUniqueID(name, "mesh"); @@ -651,15 +612,10 @@ void glTF2Exporter::ExportMeshes() p.material = mAsset->materials.Get(aim->mMaterialIndex); /******************* Vertices ********************/ - // If compression is used then you need parameters of uncompressed region: begin and size. At this step "begin" is stored. - if(comp_allow) idx_srcdata_begin = b->byteLength; - Ref v = ExportData(*mAsset, meshId, b, aim->mNumVertices, aim->mVertices, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); if (v) p.attributes.position.push_back(v); /******************** Normals ********************/ - if(comp_allow && (aim->mNormals != 0)) idx_srcdata_normal = b->byteLength;// Store index of normals array. - Ref n = ExportData(*mAsset, meshId, b, aim->mNumVertices, aim->mNormals, AttribType::VEC3, AttribType::VEC3, ComponentType_FLOAT); if (n) p.attributes.normal.push_back(n); @@ -675,16 +631,12 @@ void glTF2Exporter::ExportMeshes() if (aim->mNumUVComponents[i] > 0) { AttribType::Value type = (aim->mNumUVComponents[i] == 2) ? AttribType::VEC2 : AttribType::VEC3; - if(comp_allow) idx_srcdata_tc.push_back(b->byteLength);// Store index of texture coordinates array. - Ref tc = ExportData(*mAsset, meshId, b, aim->mNumVertices, aim->mTextureCoords[i], AttribType::VEC3, type, ComponentType_FLOAT, false); if (tc) p.attributes.texcoord.push_back(tc); } } /*************** Vertices indices ****************/ - idx_srcdata_ind = b->byteLength;// Store index of indices array. - if (aim->mNumFaces > 0) { std::vector indices; unsigned int nIndicesPerFace = aim->mFaces[0].mNumIndices; @@ -713,99 +665,7 @@ void glTF2Exporter::ExportMeshes() /*if(aim->HasBones()) { ExportSkin(*mAsset, aim, m, b, skinRef, inverseBindMatricesData); }*/ - - /****************** Compression ******************/ - ///TODO: animation: weights, joints. - if(comp_allow) - { -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - // Only one type of compression supported at now - Open3DGC. - // - o3dgc::BinaryStream bs; - o3dgc::SC3DMCEncoder encoder; - o3dgc::IndexedFaceSet comp_o3dgc_ifs; - o3dgc::SC3DMCEncodeParams comp_o3dgc_params; - - // - // Fill data for encoder. - // - // Quantization - unsigned quant_coord = mProperties->GetPropertyInteger("extensions.Open3DGC.quantization.POSITION", 12); - unsigned quant_normal = mProperties->GetPropertyInteger("extensions.Open3DGC.quantization.NORMAL", 10); - unsigned quant_texcoord = mProperties->GetPropertyInteger("extensions.Open3DGC.quantization.TEXCOORD", 10); - - // Prediction - o3dgc::O3DGCSC3DMCPredictionMode prediction_position = o3dgc::O3DGC_SC3DMC_PARALLELOGRAM_PREDICTION; - o3dgc::O3DGCSC3DMCPredictionMode prediction_normal = o3dgc::O3DGC_SC3DMC_SURF_NORMALS_PREDICTION; - o3dgc::O3DGCSC3DMCPredictionMode prediction_texcoord = o3dgc::O3DGC_SC3DMC_PARALLELOGRAM_PREDICTION; - - // IndexedFacesSet: "Crease angle", "solid", "convex" are set to default. - comp_o3dgc_ifs.SetCCW(true); - comp_o3dgc_ifs.SetIsTriangularMesh(true); - comp_o3dgc_ifs.SetNumFloatAttributes(0); - // Coordinates - comp_o3dgc_params.SetCoordQuantBits(quant_coord); - comp_o3dgc_params.SetCoordPredMode(prediction_position); - comp_o3dgc_ifs.SetNCoord(aim->mNumVertices); - comp_o3dgc_ifs.SetCoord((o3dgc::Real* const)&b->GetPointer()[idx_srcdata_begin]); - // Normals - if(idx_srcdata_normal != SIZE_MAX) - { - comp_o3dgc_params.SetNormalQuantBits(quant_normal); - comp_o3dgc_params.SetNormalPredMode(prediction_normal); - comp_o3dgc_ifs.SetNNormal(aim->mNumVertices); - comp_o3dgc_ifs.SetNormal((o3dgc::Real* const)&b->GetPointer()[idx_srcdata_normal]); - } - - // Texture coordinates - for(size_t num_tc = 0; num_tc < idx_srcdata_tc.size(); num_tc++) - { - size_t num = comp_o3dgc_ifs.GetNumFloatAttributes(); - - comp_o3dgc_params.SetFloatAttributeQuantBits(static_cast(num), quant_texcoord); - comp_o3dgc_params.SetFloatAttributePredMode(static_cast(num), prediction_texcoord); - comp_o3dgc_ifs.SetNFloatAttribute(static_cast(num), aim->mNumVertices);// number of elements. - comp_o3dgc_ifs.SetFloatAttributeDim(static_cast(num), aim->mNumUVComponents[num_tc]);// components per element: aiVector3D => x * float - comp_o3dgc_ifs.SetFloatAttributeType(static_cast(num), o3dgc::O3DGC_IFS_FLOAT_ATTRIBUTE_TYPE_TEXCOORD); - comp_o3dgc_ifs.SetFloatAttribute(static_cast(num), (o3dgc::Real* const)&b->GetPointer()[idx_srcdata_tc[num_tc]]); - comp_o3dgc_ifs.SetNumFloatAttributes(static_cast(num + 1)); - } - - // Coordinates indices - comp_o3dgc_ifs.SetNCoordIndex(aim->mNumFaces); - comp_o3dgc_ifs.SetCoordIndex((IndicesType* const)&b->GetPointer()[idx_srcdata_ind]); - // Prepare to enconding - comp_o3dgc_params.SetNumFloatAttributes(comp_o3dgc_ifs.GetNumFloatAttributes()); - if(mProperties->GetPropertyBool("extensions.Open3DGC.binary", true)) - comp_o3dgc_params.SetStreamType(o3dgc::O3DGC_STREAM_TYPE_BINARY); - else - comp_o3dgc_params.SetStreamType(o3dgc::O3DGC_STREAM_TYPE_ASCII); - - comp_o3dgc_ifs.ComputeMinMax(o3dgc::O3DGC_SC3DMC_MAX_ALL_DIMS); - // - // Encoding - // - encoder.Encode(comp_o3dgc_params, comp_o3dgc_ifs, bs); - // Replace data in buffer. - b->ReplaceData(idx_srcdata_begin, b->byteLength - idx_srcdata_begin, bs.GetBuffer(), bs.GetSize()); - // - // Add information about extension to mesh. - // - // Create extension structure. - Mesh::SCompression_Open3DGC* ext = new Mesh::SCompression_Open3DGC; - - // Fill it. - ext->Buffer = b->id; - ext->Offset = idx_srcdata_begin; - ext->Count = b->byteLength - idx_srcdata_begin; - ext->Binary = mProperties->GetPropertyBool("extensions.Open3DGC.binary"); - ext->IndicesCount = comp_o3dgc_ifs.GetNCoordIndex() * 3; - ext->VerticesCount = comp_o3dgc_ifs.GetNCoord(); - // And assign to mesh. - m->Extension.push_back(ext); -#endif - }// if(comp_allow) - }// for (unsigned int i = 0; i < mScene->mNumMeshes; ++i) + } //---------------------------------------- // Finish the skin diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 97901fc01..a2c0911bd 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -288,38 +288,6 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) for (unsigned int m = 0; m < r.meshes.Size(); ++m) { Mesh& mesh = r.meshes[m]; - // Check if mesh extensions is used - if(mesh.Extension.size() > 0) - { - for(Mesh::SExtension* cur_ext : mesh.Extension) - { -#ifdef ASSIMP_IMPORTER_GLTF_USE_OPEN3DGC - if(cur_ext->Type == Mesh::SExtension::EType::Compression_Open3DGC) - { - // Limitations for meshes when using Open3DGC-compression. - // It's a current limitation of sp... Specification have not this part still - about mesh compression. Why only one primitive? - // Because glTF is very flexibly. But in fact it ugly flexible. Every primitive can has own set of accessors and accessors can - // point to a-a-a-a-any part of buffer (through bufferview of course) and even to another buffer. We know that "Open3DGC-compression" - // is applicable only to part of buffer. As we can't guaranty continuity of the data for decoder, we will limit quantity of primitives. - // Yes indices, coordinates etc. still can br stored in different buffers, but with current specification it's a exporter problem. - // Also primitive can has only one of "POSITION", "NORMAL" and less then "AI_MAX_NUMBER_OF_TEXTURECOORDS" of "TEXCOORD". All accessor - // of primitive must point to one continuous region of the buffer. - if(mesh.primitives.size() > 2) throw DeadlyImportError("GLTF: When using Open3DGC compression then only one primitive per mesh are allowed."); - - Mesh::SCompression_Open3DGC* o3dgc_ext = (Mesh::SCompression_Open3DGC*)cur_ext; - Ref buf = r.buffers.Get(o3dgc_ext->Buffer); - - buf->EncodedRegion_SetCurrent(mesh.id); - } - else -#endif - { - throw DeadlyImportError("GLTF: Can not import mesh: unknown mesh extension (code: \"" + to_string(cur_ext->Type) + - "\"), only Open3DGC is supported."); - } - } - }// if(mesh.Extension.size() > 0) - meshOffsets.push_back(k); k += unsigned(mesh.primitives.size()); From 8bef546b41b1069effb64fd53682172fb7187607 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 31 Aug 2017 23:40:44 -0400 Subject: [PATCH 103/305] mention pbrSpecularGlossiness support --- code/glTF2Asset.h | 1 + 1 file changed, 1 insertion(+) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index dd348de1a..1f9f48e18 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * glTF Extensions Support: * KHR_binary_glTF: full + * KHR_materials_pbrSpecularGlossiness full */ #ifndef GLTF2ASSET_H_INC #define GLTF2ASSET_H_INC From feee7528d6c8323e0cdb7aeb4c38bb62909d6a51 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 02:42:09 -0400 Subject: [PATCH 104/305] Make sure `on` flag for specularGlossiness is being persisted --- code/glTF2Exporter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 2b7e4c33d..ae610d16d 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -426,6 +426,7 @@ void glTF2Exporter::ExportMaterials() KHR_materials_pbrSpecularGlossiness = true; } + m->pbrSpecularGlossiness.on = true; GetMatColor(mat, m->pbrSpecularGlossiness.diffuseFactor, "$clr.diffuse", 0, 1); GetMatColor(mat, m->pbrSpecularGlossiness.specularFactor, "$clr.specular", 0, 1); mat->Get("$mat.gltf.glossinessFactor", 0, 0, m->pbrSpecularGlossiness.glossinessFactor); From f09892ab63fcd919f1489bd8853963c020b3ca80 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 02:42:26 -0400 Subject: [PATCH 105/305] Write specularGlossiness textures on the specularGlossiness object --- code/glTF2AssetWriter.inl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index c74ea4031..9aa00729c 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -346,8 +346,8 @@ namespace glTF2 { WriteFloat(obj, m.pbrSpecularGlossiness.glossinessFactor, "glossinessFactor", w.mAl); } - WriteTex(obj, m.pbrSpecularGlossiness.diffuseTexture, "diffuseTexture", w.mAl); - WriteTex(obj, m.pbrSpecularGlossiness.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); + WriteTex(pbrSpecularGlossiness, m.pbrSpecularGlossiness.diffuseTexture, "diffuseTexture", w.mAl); + WriteTex(pbrSpecularGlossiness, m.pbrSpecularGlossiness.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); } if (!pbrSpecularGlossiness.ObjectEmpty()) { From ab08a7c3cbc27add5c10ea44786018c9b5c8ff47 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 17:49:04 -0400 Subject: [PATCH 106/305] reenable animation and skins exports Currently incorrect, however. May need to be removed --- code/glTF2Asset.h | 3 +++ code/glTF2Asset.inl | 36 ++++++++++++++++++++++++++---------- code/glTF2Exporter.cpp | 14 +++++++------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 1f9f48e18..8e3a6dd7a 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -928,9 +928,11 @@ namespace glTF2 friend class AssetWriter; typedef typename std::gltf_unordered_map< unsigned int, unsigned int > Dict; + typedef typename std::gltf_unordered_map< std::string, unsigned int > IdDict; std::vector mObjs; //! The read objects Dict mObjsByOIndex; //! The read objects accessible by original index + IdDict mObjsById; //! The read objects accessible by id const char* mDictId; //! ID of the dictionary object const char* mExtId; //! ID of the extension defining the dictionary Value* mDict; //! JSON dictionary object @@ -951,6 +953,7 @@ namespace glTF2 Ref Retrieve(unsigned int i); Ref Get(unsigned int i); + Ref Get(const char* id); Ref Create(const char* id); Ref Create(const std::string& id) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index dd110ebb7..5316b5e26 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -1,4 +1,4 @@ -/* +/* Open Asset Import Library (assimp) ---------------------------------------------------------------------- @@ -193,14 +193,6 @@ inline void LazyDict::DetachFromDocument() mDict = 0; } -template -Ref LazyDict::Get(unsigned int i) -{ - - return Ref(mObjs, i); - -} - template Ref LazyDict::Retrieve(unsigned int i) { @@ -234,12 +226,34 @@ Ref LazyDict::Retrieve(unsigned int i) return Add(inst); } +template +Ref LazyDict::Get(unsigned int i) +{ + + return Ref(mObjs, i); + +} + +template +Ref LazyDict::Get(const char* id) +{ + id = T::TranslateId(mAsset, id); + + typename IdDict::iterator it = mObjsById.find(id); + if (it != mObjsById.end()) { // already created? + return Ref(mObjs, it->second); + } + + throw std::out_of_range("id \"" + std::string(id) + "\" Doesn't exist"); +} + template Ref LazyDict::Add(T* obj) { unsigned int idx = unsigned(mObjs.size()); mObjs.push_back(obj); mObjsByOIndex[obj->oIndex] = idx; + mObjsById[obj->id] = idx; mAsset.mUsedIds[obj->id] = true; return Ref(mObjs, idx); } @@ -252,8 +266,10 @@ Ref LazyDict::Create(const char* id) throw DeadlyImportError("GLTF: two objects with the same ID exist"); } T* inst = new T(); + unsigned int idx = unsigned(mObjs.size()); inst->id = id; - inst->index = static_cast(mObjs.size()); + inst->index = idx; + inst->oIndex = idx; return Add(inst); } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index ae610d16d..164b559fd 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -121,7 +121,7 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai ExportScene(); - //ExportAnimations(); + ExportAnimations(); AssetWriter writer(*mAsset); @@ -481,7 +481,7 @@ Ref FindSkeletonRootJoint(Ref& skinRef) return parentNodeRef; } -/*void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref& bufferRef, Ref& skinRef, std::vector& inverseBindMatricesData) +void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref& bufferRef, Ref& skinRef, std::vector& inverseBindMatricesData) { if (aimesh->mNumBones < 1) { return; @@ -558,7 +558,7 @@ Ref FindSkeletonRootJoint(Ref& skinRef) delete[] jointsPerVertex; delete[] vertexWeightData; delete[] vertexJointData; -}*/ +} void glTF2Exporter::ExportMeshes() { @@ -663,9 +663,9 @@ void glTF2Exporter::ExportMeshes() } /*************** Skins ****************/ - /*if(aim->HasBones()) { + if(aim->HasBones()) { ExportSkin(*mAsset, aim, m, b, skinRef, inverseBindMatricesData); - }*/ + } } //---------------------------------------- @@ -892,7 +892,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref bufferRef = mAsset->buffers.Get(unsigned (0)); @@ -961,7 +961,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref Date: Fri, 1 Sep 2017 17:56:13 -0400 Subject: [PATCH 107/305] store node mesh vs. meshes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit glTF nodes can only hold one mesh. this simply assigns to and check’s a Node’s Mesh --- code/glTF2Asset.h | 2 +- code/glTF2Asset.inl | 9 +-------- code/glTF2AssetWriter.inl | 4 +++- code/glTF2Exporter.cpp | 19 +++++++++---------- code/glTF2Importer.cpp | 21 +++++++-------------- 5 files changed, 21 insertions(+), 34 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 8e3a6dd7a..9facef6a7 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -767,7 +767,7 @@ namespace glTF2 struct Node : public Object { std::vector< Ref > children; - std::vector< Ref > meshes; + Ref mesh; Nullable matrix; Nullable translation; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 5316b5e26..bebaeba53 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -954,16 +954,9 @@ inline void Node::Read(Value& obj, Asset& r) } if (Value* mesh = FindUInt(obj, "mesh")) { - //unsigned numMeshes = (unsigned)meshes->Size(); - unsigned numMeshes = 1; - - //std::vector meshList; - - this->meshes.reserve(numMeshes); - Ref meshRef = r.meshes.Retrieve((*mesh).GetUint()); - if (meshRef) this->meshes.push_back(meshRef); + if (meshRef) this->mesh = meshRef; } if (Value* camera = FindUInt(obj, "camera")) { diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 9aa00729c..4a2aa9f39 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -441,7 +441,9 @@ namespace glTF2 { AddRefsVector(obj, "children", n.children, w.mAl); - AddRefsVector(obj, "meshes", n.meshes, w.mAl); + if (n.mesh) { + obj.AddMember("mesh", n.mesh->index, w.mAl); + } AddRefsVector(obj, "skeletons", n.skeletons, w.mAl); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 164b559fd..7daedd4a3 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -442,16 +442,15 @@ void glTF2Exporter::ExportMaterials() */ bool FindMeshNode(Ref& nodeIn, Ref& meshNode, std::string meshID) { - for (unsigned int i = 0; i < nodeIn->meshes.size(); ++i) { - if (meshID.compare(nodeIn->meshes[i]->id) == 0) { - meshNode = nodeIn; - return true; - } + + if (nodeIn->mesh && meshID.compare(nodeIn->mesh->id) == 0) { + meshNode = nodeIn; + return true; } for (unsigned int i = 0; i < nodeIn->children.size(); ++i) { if(FindMeshNode(nodeIn->children[i], meshNode, meshID)) { - return true; + return true; } } @@ -722,8 +721,8 @@ unsigned int glTF2Exporter::ExportNodeHierarchy(const aiNode* n) CopyValue(n->mTransformation, node->matrix.value); } - for (unsigned int i = 0; i < n->mNumMeshes; ++i) { - node->meshes.push_back(mAsset->meshes.Get(n->mMeshes[i])); + if (n->mNumMeshes > 0) { + node->mesh = mAsset->meshes.Get(n->mMeshes[0]); } for (unsigned int i = 0; i < n->mNumChildren; ++i) { @@ -751,8 +750,8 @@ unsigned int glTF2Exporter::ExportNode(const aiNode* n, Ref& parent) CopyValue(n->mTransformation, node->matrix.value); } - for (unsigned int i = 0; i < n->mNumMeshes; ++i) { - node->meshes.push_back(mAsset->meshes.Get(n->mMeshes[i])); + if (n->mNumMeshes > 0) { + node->mesh = mAsset->meshes.Get(n->mMeshes[0]); } for (unsigned int i = 0; i < n->mNumChildren; ++i) { diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index a2c0911bd..90ae849bd 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -502,22 +502,15 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& } } - if (!node.meshes.empty()) { - int count = 0; - for (size_t i = 0; i < node.meshes.size(); ++i) { - int idx = node.meshes[i].GetIndex(); - count += meshOffsets[idx + 1] - meshOffsets[idx]; - } - - ainode->mNumMeshes = count; - ainode->mMeshes = new unsigned int[count]; + if (node.mesh) { + ainode->mNumMeshes = 1; + ainode->mMeshes = new unsigned int[1]; int k = 0; - for (size_t i = 0; i < node.meshes.size(); ++i) { - int idx = node.meshes[i].GetIndex(); - for (unsigned int j = meshOffsets[idx]; j < meshOffsets[idx + 1]; ++j, ++k) { - ainode->mMeshes[k] = j; - } + int idx = node.mesh.GetIndex(); + + for (unsigned int j = meshOffsets[idx]; j < meshOffsets[idx + 1]; ++j, ++k) { + ainode->mMeshes[k] = j; } } From 7f01e3f48f0008a43a64d90c94d6c8288de83498 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 17:56:42 -0400 Subject: [PATCH 108/305] Only export byteStride if not 0 --- code/glTF2AssetWriter.inl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 4a2aa9f39..69d54d97b 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -97,7 +97,11 @@ namespace glTF2 { { obj.AddMember("bufferView", a.bufferView->index, w.mAl); obj.AddMember("byteOffset", a.byteOffset, w.mAl); - obj.AddMember("byteStride", a.byteStride, w.mAl); + + if (a.byteStride != 0) { + obj.AddMember("byteStride", a.byteStride, w.mAl); + } + obj.AddMember("componentType", int(a.componentType), w.mAl); obj.AddMember("count", a.count, w.mAl); obj.AddMember("type", StringRef(AttribType::ToString(a.type)), w.mAl); From d277995a972dedcccc5258105b543b5b65fa89c9 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 17:56:48 -0400 Subject: [PATCH 109/305] Formatting --- code/glTF2Asset.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 9facef6a7..875594254 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -777,7 +777,7 @@ namespace glTF2 Ref camera; std::vector< Ref > skeletons; //!< The ID of skeleton nodes. Each of which is the root of a node hierarchy. - Ref skin; //!< The ID of the skin referenced by this node. + Ref skin; //!< The ID of the skin referenced by this node. std::string jointName; //!< Name used when this node is a joint in a skin. Ref parent; //!< This is not part of the glTF specification. Used as a helper. From 63ef19d9ad1b5a6404fb2520209bb7c0f1e05005 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 17:56:30 -0400 Subject: [PATCH 110/305] Export extensions --- code/glTF2Exporter.cpp | 18 ------------------ code/glTF2Exporter.h | 1 - code/glTF2Importer.cpp | 7 ------- 3 files changed, 26 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 7daedd4a3..9a2c1cd75 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -107,10 +107,6 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai ExportMetadata(); - if (mScene->mRootNode) { - ExportExtensions(mScene->mRootNode); - } - ExportMaterials(); if (mScene->mRootNode) { @@ -789,20 +785,6 @@ void glTF2Exporter::ExportMetadata() asset.generator = buffer; } -void glTF2Exporter::ExportExtensions(const aiNode* n) -{ - aiMetadata* mMetaData = n->mMetaData; - - if (mMetaData != nullptr) { - bool pbrSpecularGlossiness; - - if (mMetaData->Get("extensionsUsed.pbrSpecularGlossiness", pbrSpecularGlossiness)) { - mAsset->extensionsUsed.KHR_materials_pbrSpecularGlossiness = pbrSpecularGlossiness; - } - } - -} - inline void ExtractAnimationData(Asset& mAsset, std::string& animId, Ref& animRef, Ref& buffer, const aiNodeAnim* nodeChannel, float ticksPerSecond) { // Loop over the data and check to see if it exactly matches an existing buffer. diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index 827189f29..4b9fe0ce0 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -118,7 +118,6 @@ namespace Assimp void GetMatColor(const aiMaterial* mat, glTF2::vec4& prop, const char* propName, int type, int idx); void GetMatColor(const aiMaterial* mat, glTF2::vec3& prop, const char* propName, int type, int idx); void ExportMetadata(); - void ExportExtensions(const aiNode* n); void ExportMaterials(); void ExportMeshes(); unsigned int ExportNodeHierarchy(const aiNode* n); diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 90ae849bd..36fe8b70e 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -546,13 +546,6 @@ void glTF2Importer::ImportNodes(glTF2::Asset& r) //if (!mScene->mRootNode) { // mScene->mRootNode = new aiNode("EMPTY"); //} - - //initialize mMetaData; - aiMetadata* mMetaData = new aiMetadata(); - - //store used glTF extensions on the root node, for a lack of a better place. - mMetaData->Add("extensionsUsed.pbrSpecularGlossiness", r.extensionsUsed.KHR_materials_pbrSpecularGlossiness); - mScene->mRootNode->mMetaData = mMetaData; } void glTF2Importer::ImportEmbeddedTextures(glTF2::Asset& r) From 2abdbdb55e2801f284d1a8fc2a090e9da28a4557 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 1 Sep 2017 18:29:30 -0400 Subject: [PATCH 111/305] Fix unused CopyValue --- code/glTF2Importer.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 36fe8b70e..08145f5fe 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -136,10 +136,10 @@ static void CopyValue(const glTF2::vec4& v, aiColor4D& out) out.r = v[0]; out.g = v[1]; out.b = v[2]; out.a = v[3]; } -static void CopyValue(const glTF2::vec4& v, aiColor3D& out) +/*static void CopyValue(const glTF2::vec4& v, aiColor3D& out) { out.r = v[0]; out.g = v[1]; out.b = v[2]; -} +}*/ static void CopyValue(const glTF2::vec3& v, aiColor4D& out) { @@ -173,14 +173,9 @@ inline void SetMaterialColorProperty(Asset& r, vec4& prop, aiMaterial* mat, cons inline void SetMaterialColorProperty(Asset& r, vec3& prop, aiMaterial* mat, const char* pKey, unsigned int type, unsigned int idx) { - vec4 prop4; - - prop4[0] = prop[0]; - prop4[1] = prop[1]; - prop4[2] = prop[2]; - prop4[3] = 1; - - return SetMaterialColorProperty(r, prop4, mat, pKey, type, idx); + aiColor4D col; + CopyValue(prop, col); + mat->AddProperty(&col, 1, pKey, type, idx); } inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& r, glTF2::TextureInfo prop, aiMaterial* mat, aiTextureType texType, unsigned int texSlot = 0) From 7245cceead0ea647a61cebafc32191e2f588c7ea Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 3 Sep 2017 22:03:02 -0400 Subject: [PATCH 112/305] Set default values on Sampler --- code/glTF2Asset.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 875594254..711e8231b 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -800,7 +800,7 @@ namespace glTF2 SamplerWrap wrapS; //!< The texture wrapping in the S direction. (required) SamplerWrap wrapT; //!< The texture wrapping in the T direction. (required) - Sampler() {} + Sampler() { SetDefaults(); } void Read(Value& obj, Asset& r); void SetDefaults(); }; From a9c4fa84b5f48fccfc099d8c300144d5ba1807eb Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 3 Sep 2017 22:09:48 -0400 Subject: [PATCH 113/305] Sampler improvements; Add new LazyDict method Samplers are now imported into assimp internal format from textures; Samplers have defaults as per spec; Sampler enums are strongly typed with UNSET values; Samplers are exported properly; Sampler filters are exported as well; Samplers are re-used across textures on export Default sampler values are not written --- code/glTF2Asset.h | 25 ++++++----- code/glTF2Asset.inl | 28 +++++++----- code/glTF2AssetWriter.inl | 23 ++++++---- code/glTF2Exporter.cpp | 91 +++++++++++++++++++++++---------------- code/glTF2Exporter.h | 2 +- code/glTF2Importer.cpp | 21 +++++++++ 6 files changed, 123 insertions(+), 67 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 711e8231b..aabd462d8 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -242,15 +242,17 @@ namespace glTF2 }; //! Values for the Sampler::magFilter field - enum SamplerMagFilter + enum class SamplerMagFilter: unsigned int { + UNSET = 0, SamplerMagFilter_Nearest = 9728, SamplerMagFilter_Linear = 9729 }; //! Values for the Sampler::minFilter field - enum SamplerMinFilter + enum class SamplerMinFilter: unsigned int { + UNSET = 0, SamplerMinFilter_Nearest = 9728, SamplerMinFilter_Linear = 9729, SamplerMinFilter_Nearest_Mipmap_Nearest = 9984, @@ -260,11 +262,12 @@ namespace glTF2 }; //! Values for the Sampler::wrapS and Sampler::wrapT field - enum SamplerWrap + enum class SamplerWrap: unsigned int { - SamplerWrap_Clamp_To_Edge = 33071, - SamplerWrap_Mirrored_Repeat = 33648, - SamplerWrap_Repeat = 10497 + UNSET = 0, + Clamp_To_Edge = 33071, + Mirrored_Repeat = 33648, + Repeat = 10497 }; //! Values for the Texture::format and Texture::internalFormat fields @@ -795,10 +798,10 @@ namespace glTF2 struct Sampler : public Object { - SamplerMagFilter magFilter; //!< The texture magnification filter. (required) - SamplerMinFilter minFilter; //!< The texture minification filter. (required) - SamplerWrap wrapS; //!< The texture wrapping in the S direction. (required) - SamplerWrap wrapT; //!< The texture wrapping in the T direction. (required) + SamplerMagFilter magFilter; //!< The texture magnification filter. + SamplerMinFilter minFilter; //!< The texture minification filter. + SamplerWrap wrapS; //!< The texture wrapping in the S direction. + SamplerWrap wrapT; //!< The texture wrapping in the T direction. Sampler() { SetDefaults(); } void Read(Value& obj, Asset& r); @@ -955,6 +958,8 @@ namespace glTF2 Ref Get(unsigned int i); Ref Get(const char* id); + bool Has(const char* id); + Ref Create(const char* id); Ref Create(const std::string& id) { return Create(id.c_str()); } diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index bebaeba53..5764c6ff6 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -244,7 +244,17 @@ Ref LazyDict::Get(const char* id) return Ref(mObjs, it->second); } - throw std::out_of_range("id \"" + std::string(id) + "\" Doesn't exist"); + return Create(id); +} + +template +bool LazyDict::Has(const char* id) +{ + id = T::TranslateId(mAsset, id); + + typename IdDict::iterator it = mObjsById.find(id); + + return it != mObjsById.end(); } template @@ -697,6 +707,7 @@ inline void Sampler::Read(Value& obj, Asset& r) { SetDefaults(); + ReadMember(obj, "name", name); ReadMember(obj, "magFilter", magFilter); ReadMember(obj, "minFilter", minFilter); ReadMember(obj, "wrapS", wrapS); @@ -705,10 +716,11 @@ inline void Sampler::Read(Value& obj, Asset& r) inline void Sampler::SetDefaults() { - magFilter = SamplerMagFilter_Linear; - minFilter = SamplerMinFilter_Linear; - wrapS = SamplerWrap_Repeat; - wrapT = SamplerWrap_Repeat; + //only wrapping modes have defaults + wrapS = SamplerWrap::Repeat; + wrapT = SamplerWrap::Repeat; + magFilter = SamplerMagFilter::UNSET; + minFilter = SamplerMinFilter::UNSET; } inline void Texture::Read(Value& obj, Asset& r) @@ -883,7 +895,6 @@ inline void Mesh::Read(Value& pJSON_Object, Asset& pAsset_Root) // Valid attribute semantics include POSITION, NORMAL, TEXCOORD, COLOR, JOINT, JOINTMATRIX, // and WEIGHT.Attribute semantics can be of the form[semantic]_[set_index], e.g., TEXCOORD_0, TEXCOORD_1, etc. - //@TODO: update this int undPos = 0; Mesh::AccessorList* vec = 0; if (GetAttribVector(prim, attr, vec, undPos)) { @@ -943,7 +954,6 @@ inline void Node::Read(Value& obj, Asset& r) } } - if (Value* matrix = FindArray(obj, "matrix")) { ReadValue(*matrix, this->matrix); } @@ -978,7 +988,6 @@ inline void Scene::Read(Value& obj, Asset& r) } } - inline void AssetMetadata::Read(Document& doc) { // read the version, etc. @@ -1013,8 +1022,6 @@ inline void AssetMetadata::Read(Document& doc) } } - - // // Asset methods implementation // @@ -1142,7 +1149,6 @@ inline void Asset::SetAsBinary() } } - inline void Asset::ReadExtensionsUsed(Document& doc) { Value* extsUsed = FindArray(doc, "extensionsUsed"); diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 69d54d97b..f7943172e 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -467,17 +467,24 @@ namespace glTF2 { inline void Write(Value& obj, Sampler& b, AssetWriter& w) { - if (b.wrapS) { - obj.AddMember("wrapS", b.wrapS, w.mAl); + if (!b.name.empty()) { + obj.AddMember("name", b.name, w.mAl); } - if (b.wrapT) { - obj.AddMember("wrapT", b.wrapT, w.mAl); + + if (b.wrapS != SamplerWrap::UNSET && b.wrapS != SamplerWrap::Repeat) { + obj.AddMember("wrapS", static_cast(b.wrapS), w.mAl); } - if (b.magFilter) { - obj.AddMember("magFilter", b.magFilter, w.mAl); + + if (b.wrapT != SamplerWrap::UNSET && b.wrapT != SamplerWrap::Repeat) { + obj.AddMember("wrapT", static_cast(b.wrapT), w.mAl); } - if (b.minFilter) { - obj.AddMember("minFilter", b.minFilter, w.mAl); + + if (b.magFilter != SamplerMagFilter::UNSET) { + obj.AddMember("magFilter", static_cast(b.magFilter), w.mAl); + } + + if (b.minFilter != SamplerMinFilter::UNSET) { + obj.AddMember("minFilter", static_cast(b.minFilter), w.mAl); } } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 9a2c1cd75..fec7236e3 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -223,50 +223,63 @@ inline Ref ExportData(Asset& a, std::string& meshName, Ref& bu return acc; } -void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture) +inline void SetSamplerWrap(SamplerWrap& wrap, aiTextureMapMode map) { - std::string samplerId = mAsset->FindUniqueID("", "sampler"); - texture->sampler = mAsset->samplers.Create(samplerId); - - aiTextureMapMode mapU, mapV; - aiGetMaterialInteger(mat,AI_MATKEY_MAPPINGMODE_U_DIFFUSE(0),(int*)&mapU); - aiGetMaterialInteger(mat,AI_MATKEY_MAPPINGMODE_V_DIFFUSE(0),(int*)&mapV); - - switch (mapU) { - case aiTextureMapMode_Wrap: - texture->sampler->wrapS = SamplerWrap_Repeat; - break; + switch (map) { case aiTextureMapMode_Clamp: - texture->sampler->wrapS = SamplerWrap_Clamp_To_Edge; + wrap = SamplerWrap::Clamp_To_Edge; break; case aiTextureMapMode_Mirror: - texture->sampler->wrapS = SamplerWrap_Mirrored_Repeat; + wrap = SamplerWrap::Mirrored_Repeat; break; - case aiTextureMapMode_Decal: - default: - texture->sampler->wrapS = SamplerWrap_Repeat; - break; - }; - - switch (mapV) { case aiTextureMapMode_Wrap: - texture->sampler->wrapT = SamplerWrap_Repeat; - break; - case aiTextureMapMode_Clamp: - texture->sampler->wrapT = SamplerWrap_Clamp_To_Edge; - break; - case aiTextureMapMode_Mirror: - texture->sampler->wrapT = SamplerWrap_Mirrored_Repeat; - break; case aiTextureMapMode_Decal: default: - texture->sampler->wrapT = SamplerWrap_Repeat; + wrap = SamplerWrap::Repeat; break; }; +} - // Hard coded Texture filtering options because I do not know where to find them in the aiMaterial. - texture->sampler->magFilter = SamplerMagFilter_Linear; - texture->sampler->minFilter = SamplerMinFilter_Linear_Mipmap_Linear; +void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture, aiTextureType tt, unsigned int slot) +{ + aiString aId; + std::string id; + if (aiGetMaterialString(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "id").c_str(), tt, slot, &aId) == AI_SUCCESS) { + id = aId.C_Str(); + } + + if (mAsset->samplers.Has(id.c_str())) { + texture->sampler = mAsset->samplers.Get(id.c_str()); + } else { + id = mAsset->FindUniqueID(id, "sampler"); + + texture->sampler = mAsset->samplers.Create(id.c_str()); + + aiTextureMapMode mapU, mapV; + SamplerMagFilter filterMag; + SamplerMinFilter filterMin; + + if (aiGetMaterialInteger(mat, AI_MATKEY_MAPPINGMODE_U(tt, slot), (int*)&mapU) == AI_SUCCESS) { + SetSamplerWrap(texture->sampler->wrapS, mapU); + } + + if (aiGetMaterialInteger(mat, AI_MATKEY_MAPPINGMODE_V(tt, slot), (int*)&mapV) == AI_SUCCESS) { + SetSamplerWrap(texture->sampler->wrapT, mapV); + } + + if (aiGetMaterialInteger(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermag").c_str(), tt, slot, (int*)&filterMag) == AI_SUCCESS) { + texture->sampler->magFilter = filterMag; + } + + if (aiGetMaterialInteger(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermin").c_str(), tt, slot, (int*)&filterMin) == AI_SUCCESS) { + texture->sampler->minFilter = filterMin; + } + + aiString name; + if (aiGetMaterialString(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "name").c_str(), tt, slot, &name) == AI_SUCCESS) { + texture->sampler->name = name.C_Str(); + } + } } void glTF2Exporter::GetMatTexProp(const aiMaterial* mat, unsigned int& prop, const char* propName, aiTextureType tt, unsigned int slot) @@ -324,7 +337,7 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe texture->source->uri = path; } - GetTexSampler(mat, texture); + GetTexSampler(mat, texture, tt, slot); } } } @@ -390,17 +403,23 @@ void glTF2Exporter::ExportMaterials() for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { const aiMaterial* mat = mScene->mMaterials[i]; + std::string id = "material_" + std::to_string(i); + + Ref m = mAsset->materials.Create(id); + std::string name; if (mat->Get(AI_MATKEY_NAME, aiName) == AI_SUCCESS) { name = aiName.C_Str(); } name = mAsset->FindUniqueID(name, "material"); - Ref m = mAsset->materials.Create(name); + m->name = name; GetMatTex(mat, m->pbrMetallicRoughness.baseColorTexture, aiTextureType_DIFFUSE); GetMatTex(mat, m->pbrMetallicRoughness.metallicRoughnessTexture, aiTextureType_UNKNOWN, 0);//get unknown slot GetMatColor(mat, m->pbrMetallicRoughness.baseColorFactor, AI_MATKEY_COLOR_DIFFUSE); + mat->Get("$mat.gltf.pbrMetallicRoughness.metallicFactor", 0, 0, m->pbrMetallicRoughness.metallicFactor); + mat->Get("$mat.gltf.pbrMetallicRoughness.roughnessFactor", 0, 0, m->pbrMetallicRoughness.roughnessFactor); GetMatTex(mat, m->normalTexture, aiTextureType_NORMALS); GetMatTex(mat, m->occlusionTexture, aiTextureType_LIGHTMAP); @@ -409,8 +428,6 @@ void glTF2Exporter::ExportMaterials() mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); mat->Get("$mat.gltf.alphaCutoff", 0, 0, m->alphaCutoff); - mat->Get("$mat.gltf.metallicFactor", 0, 0, m->pbrMetallicRoughness.metallicFactor); - mat->Get("$mat.gltf.roughnessFactor", 0, 0, m->pbrMetallicRoughness.roughnessFactor); mat->Get("$mat.gltf.alphaMode", 0, 0, m->alphaMode); bool hasPbrSpecularGlossiness; diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index 4b9fe0ce0..420c2afa6 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -108,7 +108,7 @@ namespace Assimp void WriteBinaryData(IOStream* outfile, std::size_t sceneLength); - void GetTexSampler(const aiMaterial* mat, glTF2::Ref texture); + void GetTexSampler(const aiMaterial* mat, glTF2::Ref texture, aiTextureType tt, unsigned int slot); void GetMatTexProp(const aiMaterial* mat, unsigned int& prop, const char* propName, aiTextureType tt, unsigned int idx); void GetMatTexProp(const aiMaterial* mat, float& prop, const char* propName, aiTextureType tt, unsigned int idx); void GetMatTex(const aiMaterial* mat, glTF2::Ref& texture, aiTextureType tt, unsigned int slot); diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 08145f5fe..86301a3f4 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -194,6 +194,27 @@ inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& const char *texCoordName = (std::string(_AI_MATKEY_TEXTURE_BASE) + ".texCoord").c_str(); mat->AddProperty(&prop.texCoord, 1, texCoordName, texType, texSlot); + + if (prop.texture->sampler) { + Ref sampler = prop.texture->sampler; + + aiString name(sampler->name); + aiString id(sampler->id); + + mat->AddProperty(&name, (std::string(_AI_MATKEY_MAPPING_BASE) + "name").c_str(), texType, texSlot); + mat->AddProperty(&id, (std::string(_AI_MATKEY_MAPPING_BASE) + "id").c_str(), texType, texSlot); + + mat->AddProperty(&sampler->wrapS, 1, AI_MATKEY_MAPPINGMODE_U(texType, texSlot)); + mat->AddProperty(&sampler->wrapT, 1, AI_MATKEY_MAPPINGMODE_V(texType, texSlot)); + + if (sampler->magFilter != SamplerMagFilter::UNSET) { + mat->AddProperty(&sampler->magFilter, 1, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermag").c_str(), texType, texSlot); + } + + if (sampler->minFilter != SamplerMinFilter::UNSET) { + mat->AddProperty(&sampler->minFilter, 1, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermin").c_str(), texType, texSlot); + } + } } } From 37527849b771552c9003dc488a5a97d76a196123 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 3 Sep 2017 22:11:20 -0400 Subject: [PATCH 114/305] Export material names properly --- code/glTF2AssetWriter.inl | 4 ---- code/glTF2Importer.cpp | 8 +++++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index f7943172e..c10a6a08b 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -284,10 +284,6 @@ namespace glTF2 { inline void Write(Value& obj, Material& m, AssetWriter& w) { - if (!m.name.empty()) { - obj.AddMember("name", m.name, w.mAl); - } - Value pbrMetallicRoughness; pbrMetallicRoughness.SetObject(); { diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 86301a3f4..82ea5f3e7 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -125,7 +125,6 @@ bool glTF2Importer::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool } - //static void CopyValue(const glTF2::vec3& v, aiColor3D& out) //{ // out.r = v[0]; out.g = v[1]; out.b = v[2]; @@ -228,8 +227,11 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) Material& mat = r.materials[i]; - aiString str(mat.id); - aimat->AddProperty(&str, AI_MATKEY_NAME); + if (!mat.name.empty()) { + aiString str(mat.name); + + aimat->AddProperty(&str, AI_MATKEY_NAME); + } SetMaterialColorProperty(r, mat.pbrMetallicRoughness.baseColorFactor, aimat, AI_MATKEY_COLOR_DIFFUSE); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrMetallicRoughness.baseColorTexture, aimat, aiTextureType_DIFFUSE); From 21259e08357c0b8487ab33ffd87d5f9118420dd7 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 14:39:52 -0400 Subject: [PATCH 115/305] Use different form of index accessor --- code/glTF2Asset.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 5764c6ff6..88dbfcfdb 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -211,7 +211,7 @@ Ref LazyDict::Retrieve(unsigned int i) throw DeadlyImportError("GLTF: Field is not an array \"" + std::string(mDictId) + "\""); } - Value& obj = mDict->operator[](i); + Value &obj = (*mDict)[i]; if (!obj.IsObject()) { throw DeadlyImportError("GLTF: Object at index \"" + std::to_string(i) + "\" is not a JSON object"); From 1a5823700f17caa6c6cdedfd34ce5bbd3ba85556 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 14:40:27 -0400 Subject: [PATCH 116/305] Remove need for Has by returning an empty Ref in Get --- code/glTF2Asset.h | 2 -- code/glTF2Asset.inl | 12 +----------- code/glTF2Exporter.cpp | 4 ++-- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index aabd462d8..9a70ff4ff 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -958,8 +958,6 @@ namespace glTF2 Ref Get(unsigned int i); Ref Get(const char* id); - bool Has(const char* id); - Ref Create(const char* id); Ref Create(const std::string& id) { return Create(id.c_str()); } diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 88dbfcfdb..1c75d0040 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -244,17 +244,7 @@ Ref LazyDict::Get(const char* id) return Ref(mObjs, it->second); } - return Create(id); -} - -template -bool LazyDict::Has(const char* id) -{ - id = T::TranslateId(mAsset, id); - - typename IdDict::iterator it = mObjsById.find(id); - - return it != mObjsById.end(); + return Ref(); } template diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index fec7236e3..073309b06 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -248,8 +248,8 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture, a id = aId.C_Str(); } - if (mAsset->samplers.Has(id.c_str())) { - texture->sampler = mAsset->samplers.Get(id.c_str()); + if (Ref ref = mAsset->samplers.Get(id.c_str())) { + texture->sampler = ref; } else { id = mAsset->FindUniqueID(id, "sampler"); From 03cfa04ee4a4ef85a3eeefa879bec4e088418287 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 15:45:32 -0400 Subject: [PATCH 117/305] Define default material values as static constants for reuse --- code/glTF2Asset.h | 10 +++++++--- code/glTF2Asset.inl | 16 ++++++++-------- code/glTF2AssetWriter.inl | 14 ++------------ 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 9a70ff4ff..486e23bc0 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -135,8 +135,7 @@ namespace glTF2 // Vec/matrix types, as raw float arrays typedef float (vec3)[3]; typedef float (vec4)[4]; - typedef float (mat4)[16]; - + typedef float (mat4)[16]; namespace Util { @@ -670,7 +669,12 @@ namespace glTF2 inline uint8_t* StealData(); inline void SetData(uint8_t* data, size_t length, Asset& r); - }; + }; + + const vec4 defaultBaseColor = {1, 1, 1, 1}; + const vec3 defaultEmissiveFactor = {0, 0, 0}; + const vec4 defaultDiffuseFactor = {1, 1, 1, 1}; + const vec3 defaultSpecularFactor = {1, 1, 1}; struct TextureInfo { diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 1c75d0040..07e0a5e8e 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -803,28 +803,28 @@ inline void Material::Read(Value& material, Asset& r) } namespace { - void SetVector(vec4& v, float x, float y, float z, float w) - { v[0] = x; v[1] = y; v[2] = z; v[3] = w; } + void SetVector(vec4& v, const float(&in)[4]) + { v[0] = in[0]; v[1] = in[1]; v[2] = in[2]; v[3] = in[3]; } - void SetVector(vec3& v, float x, float y, float z) - { v[0] = x; v[1] = y; v[2] = z; } + void SetVector(vec3& v, const float(&in)[3]) + { v[0] = in[0]; v[1] = in[1]; v[2] = in[2]; } } inline void Material::SetDefaults() { //pbr materials - SetVector(pbrMetallicRoughness.baseColorFactor, 1, 1, 1, 1); + SetVector(pbrMetallicRoughness.baseColorFactor, defaultBaseColor); pbrMetallicRoughness.metallicFactor = 1.0; pbrMetallicRoughness.roughnessFactor = 1.0; - SetVector(emissiveFactor, 0, 0, 0); + SetVector(emissiveFactor, defaultEmissiveFactor); alphaMode = "OPAQUE"; alphaCutoff = 0.5; doubleSided = false; //pbrSpecularGlossiness properties - SetVector(pbrSpecularGlossiness.diffuseFactor, 1, 1, 1, 1); - SetVector(pbrSpecularGlossiness.specularFactor, 1, 1, 1); + SetVector(pbrSpecularGlossiness.diffuseFactor, defaultDiffuseFactor); + SetVector(pbrSpecularGlossiness.specularFactor, defaultSpecularFactor); pbrSpecularGlossiness.glossinessFactor = 1.0; } diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index c10a6a08b..475c7a0d9 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -268,7 +268,7 @@ namespace glTF2 { } template - inline void WriteVec(Value& obj, float(&prop)[N], const char* propName, float(&defaultVal)[N], MemoryPoolAllocator<>& al) + inline void WriteVec(Value& obj, float(&prop)[N], const char* propName, const float(&defaultVal)[N], MemoryPoolAllocator<>& al) { if (!std::equal(std::begin(prop), std::end(prop), std::begin(defaultVal))) { WriteVec(obj, prop, propName, al); @@ -289,10 +289,7 @@ namespace glTF2 { { WriteTex(pbrMetallicRoughness, m.pbrMetallicRoughness.baseColorTexture, "baseColorTexture", w.mAl); WriteTex(pbrMetallicRoughness, m.pbrMetallicRoughness.metallicRoughnessTexture, "metallicRoughnessTexture", w.mAl); - - //@TODO: define this as a constant? - vec4 defaultEmissiveFactor = {1, 1, 1, 1}; - WriteVec(pbrMetallicRoughness, m.pbrMetallicRoughness.baseColorFactor, "baseColorFactor", defaultEmissiveFactor, w.mAl); + WriteVec(pbrMetallicRoughness, m.pbrMetallicRoughness.baseColorFactor, "baseColorFactor", defaultBaseColor, w.mAl); if (m.pbrMetallicRoughness.metallicFactor != 1) { WriteFloat(pbrMetallicRoughness, m.pbrMetallicRoughness.metallicFactor, "metallicFactor", w.mAl); @@ -310,9 +307,6 @@ namespace glTF2 { WriteTex(obj, m.normalTexture, "normalTexture", w.mAl); WriteTex(obj, m.emissiveTexture, "emissiveTexture", w.mAl); WriteTex(obj, m.occlusionTexture, "occlusionTexture", w.mAl); - - //@TODO: define this as a constant? - vec3 defaultEmissiveFactor = {0, 0, 0}; WriteVec(obj, m.emissiveFactor, "emissiveFactor", defaultEmissiveFactor, w.mAl); if (m.alphaCutoff != 0.5) { @@ -335,11 +329,7 @@ namespace glTF2 { pbrSpecularGlossiness.SetObject(); { //pbrSpecularGlossiness - - vec4 defaultDiffuseFactor = {1, 1, 1, 1}; WriteVec(pbrSpecularGlossiness, m.pbrSpecularGlossiness.diffuseFactor, "diffuseFactor", defaultDiffuseFactor, w.mAl); - - vec3 defaultSpecularFactor = {1, 1, 1}; WriteVec(pbrSpecularGlossiness, m.pbrSpecularGlossiness.specularFactor, "specularFactor", defaultSpecularFactor, w.mAl); if (m.pbrSpecularGlossiness.glossinessFactor != 1) { From 44757af34a0047e974ba6d30b9b027d0fffecc5d Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 16:29:00 -0400 Subject: [PATCH 118/305] Implement pbrSpecularGlossiness property as Nullable --- code/glTF2Asset.h | 8 +++++--- code/glTF2Asset.inl | 23 ++++++++++++++--------- code/glTF2AssetWriter.inl | 21 +++++++++++---------- code/glTF2Exporter.cpp | 15 +++++++++------ code/glTF2Importer.cpp | 14 +++++++------- 5 files changed, 46 insertions(+), 35 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 486e23bc0..07f8614ab 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -711,12 +711,14 @@ namespace glTF2 struct PbrSpecularGlossiness { - bool on = false; vec4 diffuseFactor; vec3 specularFactor; float glossinessFactor; TextureInfo diffuseTexture; - TextureInfo specularGlossinessTexture; + TextureInfo specularGlossinessTexture; + + PbrSpecularGlossiness() { SetDefaults(); } + void SetDefaults(); }; //! The material appearance of a primitive. @@ -735,7 +737,7 @@ namespace glTF2 bool doubleSided; //extension: KHR_materials_pbrSpecularGlossiness - PbrSpecularGlossiness pbrSpecularGlossiness; + Nullable pbrSpecularGlossiness; Material() { SetDefaults(); } void Read(Value& obj, Asset& r); diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 07e0a5e8e..e2d812563 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -790,13 +790,15 @@ inline void Material::Read(Value& material, Asset& r) if (Value* extensions = FindObject(material, "extensions")) { if (r.extensionsUsed.KHR_materials_pbrSpecularGlossiness) { if (Value* pbrSpecularGlossiness = FindObject(*extensions, "KHR_materials_pbrSpecularGlossiness")) { - this->pbrSpecularGlossiness.on = true; + PbrSpecularGlossiness pbrSG; - ReadMember(*pbrSpecularGlossiness, "diffuseFactor", this->pbrSpecularGlossiness.diffuseFactor); - ReadTextureProperty(r, *pbrSpecularGlossiness, "diffuseTexture", this->pbrSpecularGlossiness.diffuseTexture); - ReadTextureProperty(r, *pbrSpecularGlossiness, "specularGlossinessTexture", this->pbrSpecularGlossiness.specularGlossinessTexture); - ReadMember(*pbrSpecularGlossiness, "specularFactor", this->pbrSpecularGlossiness.specularFactor); - ReadMember(*pbrSpecularGlossiness, "glossinessFactor", this->pbrSpecularGlossiness.glossinessFactor); + ReadMember(*pbrSpecularGlossiness, "diffuseFactor", pbrSG.diffuseFactor); + ReadTextureProperty(r, *pbrSpecularGlossiness, "diffuseTexture", pbrSG.diffuseTexture); + ReadTextureProperty(r, *pbrSpecularGlossiness, "specularGlossinessTexture", pbrSG.specularGlossinessTexture); + ReadMember(*pbrSpecularGlossiness, "specularFactor", pbrSG.specularFactor); + ReadMember(*pbrSpecularGlossiness, "glossinessFactor", pbrSG.glossinessFactor); + + this->pbrSpecularGlossiness = Nullable(pbrSG); } } } @@ -821,11 +823,14 @@ inline void Material::SetDefaults() alphaMode = "OPAQUE"; alphaCutoff = 0.5; doubleSided = false; +} +inline void PbrSpecularGlossiness::SetDefaults() +{ //pbrSpecularGlossiness properties - SetVector(pbrSpecularGlossiness.diffuseFactor, defaultDiffuseFactor); - SetVector(pbrSpecularGlossiness.specularFactor, defaultSpecularFactor); - pbrSpecularGlossiness.glossinessFactor = 1.0; + SetVector(diffuseFactor, defaultDiffuseFactor); + SetVector(specularFactor, defaultSpecularFactor); + glossinessFactor = 1.0; } namespace { diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 475c7a0d9..ad68a14e1 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -324,22 +324,23 @@ namespace glTF2 { Value exts; exts.SetObject(); - if (m.pbrSpecularGlossiness.on) { + if (m.pbrSpecularGlossiness.isPresent) { Value pbrSpecularGlossiness; pbrSpecularGlossiness.SetObject(); - { - //pbrSpecularGlossiness - WriteVec(pbrSpecularGlossiness, m.pbrSpecularGlossiness.diffuseFactor, "diffuseFactor", defaultDiffuseFactor, w.mAl); - WriteVec(pbrSpecularGlossiness, m.pbrSpecularGlossiness.specularFactor, "specularFactor", defaultSpecularFactor, w.mAl); - if (m.pbrSpecularGlossiness.glossinessFactor != 1) { - WriteFloat(obj, m.pbrSpecularGlossiness.glossinessFactor, "glossinessFactor", w.mAl); - } + PbrSpecularGlossiness &pbrSG = m.pbrSpecularGlossiness.value; - WriteTex(pbrSpecularGlossiness, m.pbrSpecularGlossiness.diffuseTexture, "diffuseTexture", w.mAl); - WriteTex(pbrSpecularGlossiness, m.pbrSpecularGlossiness.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); + //pbrSpecularGlossiness + WriteVec(pbrSpecularGlossiness, pbrSG.diffuseFactor, "diffuseFactor", defaultDiffuseFactor, w.mAl); + WriteVec(pbrSpecularGlossiness, pbrSG.specularFactor, "specularFactor", defaultSpecularFactor, w.mAl); + + if (pbrSG.glossinessFactor != 1) { + WriteFloat(obj, pbrSG.glossinessFactor, "glossinessFactor", w.mAl); } + WriteTex(pbrSpecularGlossiness, pbrSG.diffuseTexture, "diffuseTexture", w.mAl); + WriteTex(pbrSpecularGlossiness, pbrSG.specularGlossinessTexture, "specularGlossinessTexture", w.mAl); + if (!pbrSpecularGlossiness.ObjectEmpty()) { exts.AddMember("KHR_materials_pbrSpecularGlossiness", pbrSpecularGlossiness, w.mAl); } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 073309b06..600e2b3d4 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -439,12 +439,15 @@ void glTF2Exporter::ExportMaterials() KHR_materials_pbrSpecularGlossiness = true; } - m->pbrSpecularGlossiness.on = true; - GetMatColor(mat, m->pbrSpecularGlossiness.diffuseFactor, "$clr.diffuse", 0, 1); - GetMatColor(mat, m->pbrSpecularGlossiness.specularFactor, "$clr.specular", 0, 1); - mat->Get("$mat.gltf.glossinessFactor", 0, 0, m->pbrSpecularGlossiness.glossinessFactor); - GetMatTex(mat, m->pbrSpecularGlossiness.diffuseTexture, aiTextureType_DIFFUSE, 1); - GetMatTex(mat, m->pbrSpecularGlossiness.specularGlossinessTexture, aiTextureType_UNKNOWN, 1); + PbrSpecularGlossiness pbrSG; + + GetMatColor(mat, pbrSG.diffuseFactor, "$clr.diffuse", 0, 1); + GetMatColor(mat, pbrSG.specularFactor, "$clr.specular", 0, 1); + mat->Get("$mat.gltf.glossinessFactor", 0, 0, pbrSG.glossinessFactor); + GetMatTex(mat, pbrSG.diffuseTexture, aiTextureType_DIFFUSE, 1); + GetMatTex(mat, pbrSG.specularGlossinessTexture, aiTextureType_UNKNOWN, 1); + + m->pbrSpecularGlossiness = Nullable(pbrSG); } } } diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 82ea5f3e7..ff806257f 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -249,13 +249,13 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) aimat->AddProperty(&mat.alphaCutoff, 1, "$mat.gltf.alphaCutoff"); //pbrSpecularGlossiness - if (mat.pbrSpecularGlossiness.on) { - aimat->AddProperty(&mat.pbrSpecularGlossiness.on, 1, "$mat.gltf.pbrSpecularGlossiness.on"); - SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.diffuseFactor, aimat, "$clr.diffuse", 0, 1); - SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.specularFactor, aimat, "$clr.specular", 0, 1); - aimat->AddProperty(&mat.pbrSpecularGlossiness.glossinessFactor, 1, "$mat.gltf.pbrSpecularGlossiness.glossinessFactor"); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.diffuseTexture, aimat, aiTextureType_DIFFUSE, 1); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.specularGlossinessTexture, aimat, aiTextureType_UNKNOWN, 1); + if (mat.pbrSpecularGlossiness.isPresent) { + aimat->AddProperty(&mat.pbrSpecularGlossiness.isPresent, 1, "$mat.gltf.pbrSpecularGlossiness.on"); + SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.value.diffuseFactor, aimat, "$clr.diffuse", 0, 1); + SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.value.specularFactor, aimat, "$clr.specular", 0, 1); + aimat->AddProperty(&mat.pbrSpecularGlossiness.value.glossinessFactor, 1, "$mat.gltf.pbrSpecularGlossiness.glossinessFactor"); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.value.diffuseTexture, aimat, aiTextureType_DIFFUSE, 1); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.value.specularGlossinessTexture, aimat, aiTextureType_UNKNOWN, 1); } } } From 54dd4804cda6f5f6a6e08781ee4e42325f12755a Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 16:39:34 -0400 Subject: [PATCH 119/305] Fix indentation --- code/glTF2Importer.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index ff806257f..742088042 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -306,7 +306,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) for (unsigned int m = 0; m < r.meshes.Size(); ++m) { Mesh& mesh = r.meshes[m]; - meshOffsets.push_back(k); + meshOffsets.push_back(k); k += unsigned(mesh.primitives.size()); for (unsigned int p = 0; p < mesh.primitives.size(); ++p) { @@ -339,16 +339,17 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) case PrimitiveMode_TRIANGLE_FAN: aim->mPrimitiveTypes |= aiPrimitiveType_TRIANGLE; break; + } Mesh::Primitive::Attributes& attr = prim.attributes; - if (attr.position.size() > 0 && attr.position[0]) { + if (attr.position.size() > 0 && attr.position[0]) { aim->mNumVertices = attr.position[0]->count; attr.position[0]->ExtractData(aim->mVertices); - } + } - if (attr.normal.size() > 0 && attr.normal[0]) attr.normal[0]->ExtractData(aim->mNormals); + if (attr.normal.size() > 0 && attr.normal[0]) attr.normal[0]->ExtractData(aim->mNormals); for (size_t tc = 0; tc < attr.texcoord.size() && tc < AI_MAX_NUMBER_OF_TEXTURECOORDS; ++tc) { attr.texcoord[tc]->ExtractData(aim->mTextureCoords[tc]); @@ -362,7 +363,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) if (prim.indices) { - aiFace* faces = 0; + aiFace* faces = 0; unsigned int nFaces = 0; unsigned int count = prim.indices->count; @@ -638,7 +639,7 @@ void glTF2Importer::InternReadFile(const std::string& pFile, aiScene* pScene, IO // TODO: it does not split the loaded vertices, should it? //pScene->mFlags |= AI_SCENE_FLAGS_NON_VERBOSE_FORMAT; - MakeVerboseFormatProcess process; + MakeVerboseFormatProcess process; process.Execute(pScene); From 3ba00ca4219ea12fdcab9f20c32720cb533b26b2 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 18:04:09 -0400 Subject: [PATCH 120/305] Define gltf material property names as constants --- code/glTF2Asset.h | 26 +++++++++++++++++++++++++- code/glTF2Exporter.cpp | 30 +++++++++++++++--------------- code/glTF2Importer.cpp | 36 ++++++++++++++++++------------------ 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 07f8614ab..b4369fe71 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -163,7 +163,31 @@ namespace glTF2 //! Magic number for GLB files - #define AI_GLB_MAGIC_NUMBER "glTF" + #define AI_GLB_MAGIC_NUMBER "glTF" + + #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0,0 + #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0, 0 + #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLICROUGHNESS_TEXTURE aiTextureType_UNKNOWN, 0 + #define AI_MATKEY_GLTF_ALPHAMODE "$mat.gltf.alphaMode", 0, 0 + #define AI_MATKEY_GLTF_ALPHACUTOFF "$mat.gltf.alphaCutoff", 0, 0 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS "$mat.gltf.pbrSpecularGlossiness", 0, 0 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_FACTOR "$clr.diffuse", 0, 1 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULAR_FACTOR "$clr.specular", 0, 1 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_GLOSSINESS_FACTOR "$mat.gltf.pbrMetallicRoughness.glossinessFactor", 0,0 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_TEXTURE aiTextureType_DIFFUSE, 1 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULARGLOSSINESS_TEXTURE aiTextureType_UNKNOWN, 1 + + #define _AI_MATKEY_GLTF_TEXTURE_TEXCOORD_BASE (std::string(_AI_MATKEY_TEXTURE_BASE) + ".texCoord").c_str() + #define _AI_MATKEY_GLTF_MAPPINGNAME_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "name").c_str() + #define _AI_MATKEY_GLTF_MAPPINGID_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "id").c_str() + #define _AI_MATKEY_GLTF_MAPPINGFILTER_MAG_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermag").c_str() + #define _AI_MATKEY_GLTF_MAPPINGFILTER_MIN_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermin").c_str() + + #define AI_MATKEY_GLTF_TEXTURE_TEXCOORD _AI_MATKEY_GLTF_TEXTURE_TEXCOORD_BASE, type, N + #define AI_MATKEY_GLTF_MAPPINGNAME(type, N) _AI_MATKEY_GLTF_MAPPINGNAME_BASE, type, N + #define AI_MATKEY_GLTF_MAPPINGID(type, N) _AI_MATKEY_GLTF_MAPPINGID_BASE, type, N + #define AI_MATKEY_GLTF_MAPPINGFILTER_MAG(type, N) _AI_MATKEY_GLTF_MAPPINGFILTER_MAG_BASE, type, N + #define AI_MATKEY_GLTF_MAPPINGFILTER_MIN(type, N) _AI_MATKEY_GLTF_MAPPINGFILTER_MIN_BASE, type, N #ifdef ASSIMP_API #include "./../include/assimp/Compiler/pushpack1.h" diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 600e2b3d4..931e491e3 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -244,7 +244,7 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture, a { aiString aId; std::string id; - if (aiGetMaterialString(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "id").c_str(), tt, slot, &aId) == AI_SUCCESS) { + if (aiGetMaterialString(mat, AI_MATKEY_GLTF_MAPPINGID(tt, slot), &aId) == AI_SUCCESS) { id = aId.C_Str(); } @@ -267,16 +267,16 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture, a SetSamplerWrap(texture->sampler->wrapT, mapV); } - if (aiGetMaterialInteger(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermag").c_str(), tt, slot, (int*)&filterMag) == AI_SUCCESS) { + if (aiGetMaterialInteger(mat, AI_MATKEY_GLTF_MAPPINGFILTER_MAG(tt, slot), (int*)&filterMag) == AI_SUCCESS) { texture->sampler->magFilter = filterMag; } - if (aiGetMaterialInteger(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermin").c_str(), tt, slot, (int*)&filterMin) == AI_SUCCESS) { + if (aiGetMaterialInteger(mat, AI_MATKEY_GLTF_MAPPINGFILTER_MIN(tt, slot), (int*)&filterMin) == AI_SUCCESS) { texture->sampler->minFilter = filterMin; } aiString name; - if (aiGetMaterialString(mat, (std::string(_AI_MATKEY_MAPPING_BASE) + "name").c_str(), tt, slot, &name) == AI_SUCCESS) { + if (aiGetMaterialString(mat, AI_MATKEY_GLTF_MAPPINGNAME(tt, slot), &name) == AI_SUCCESS) { texture->sampler->name = name.C_Str(); } } @@ -416,10 +416,10 @@ void glTF2Exporter::ExportMaterials() m->name = name; GetMatTex(mat, m->pbrMetallicRoughness.baseColorTexture, aiTextureType_DIFFUSE); - GetMatTex(mat, m->pbrMetallicRoughness.metallicRoughnessTexture, aiTextureType_UNKNOWN, 0);//get unknown slot + GetMatTex(mat, m->pbrMetallicRoughness.metallicRoughnessTexture, AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLICROUGHNESS_TEXTURE); GetMatColor(mat, m->pbrMetallicRoughness.baseColorFactor, AI_MATKEY_COLOR_DIFFUSE); - mat->Get("$mat.gltf.pbrMetallicRoughness.metallicFactor", 0, 0, m->pbrMetallicRoughness.metallicFactor); - mat->Get("$mat.gltf.pbrMetallicRoughness.roughnessFactor", 0, 0, m->pbrMetallicRoughness.roughnessFactor); + mat->Get(AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR, m->pbrMetallicRoughness.metallicFactor); + mat->Get(AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR, m->pbrMetallicRoughness.roughnessFactor); GetMatTex(mat, m->normalTexture, aiTextureType_NORMALS); GetMatTex(mat, m->occlusionTexture, aiTextureType_LIGHTMAP); @@ -427,11 +427,11 @@ void glTF2Exporter::ExportMaterials() GetMatColor(mat, m->emissiveFactor, AI_MATKEY_COLOR_EMISSIVE); mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); - mat->Get("$mat.gltf.alphaCutoff", 0, 0, m->alphaCutoff); - mat->Get("$mat.gltf.alphaMode", 0, 0, m->alphaMode); + mat->Get(AI_MATKEY_GLTF_ALPHACUTOFF, m->alphaCutoff); + mat->Get(AI_MATKEY_GLTF_ALPHAMODE, m->alphaMode); bool hasPbrSpecularGlossiness; - mat->Get("$mat.gltf.pbrSpecularGlossiness.on", 0, 0, hasPbrSpecularGlossiness); + mat->Get(AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS, hasPbrSpecularGlossiness); if (hasPbrSpecularGlossiness) { @@ -441,11 +441,11 @@ void glTF2Exporter::ExportMaterials() PbrSpecularGlossiness pbrSG; - GetMatColor(mat, pbrSG.diffuseFactor, "$clr.diffuse", 0, 1); - GetMatColor(mat, pbrSG.specularFactor, "$clr.specular", 0, 1); - mat->Get("$mat.gltf.glossinessFactor", 0, 0, pbrSG.glossinessFactor); - GetMatTex(mat, pbrSG.diffuseTexture, aiTextureType_DIFFUSE, 1); - GetMatTex(mat, pbrSG.specularGlossinessTexture, aiTextureType_UNKNOWN, 1); + GetMatColor(mat, pbrSG.diffuseFactor, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_FACTOR); + GetMatColor(mat, pbrSG.specularFactor, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULAR_FACTOR); + mat->Get(AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_GLOSSINESS_FACTOR, pbrSG.glossinessFactor); + GetMatTex(mat, pbrSG.diffuseTexture, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_TEXTURE); + GetMatTex(mat, pbrSG.specularGlossinessTexture, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULARGLOSSINESS_TEXTURE); m->pbrSpecularGlossiness = Nullable(pbrSG); } diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 742088042..860974243 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -190,9 +190,7 @@ inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& } mat->AddProperty(&uri, AI_MATKEY_TEXTURE(texType, texSlot)); - - const char *texCoordName = (std::string(_AI_MATKEY_TEXTURE_BASE) + ".texCoord").c_str(); - mat->AddProperty(&prop.texCoord, 1, texCoordName, texType, texSlot); + mat->AddProperty(&prop.texCoord, 1, _AI_MATKEY_GLTF_TEXTURE_TEXCOORD_BASE, texType, texSlot); if (prop.texture->sampler) { Ref sampler = prop.texture->sampler; @@ -200,18 +198,18 @@ inline void SetMaterialTextureProperty(std::vector& embeddedTexIdxs, Asset& aiString name(sampler->name); aiString id(sampler->id); - mat->AddProperty(&name, (std::string(_AI_MATKEY_MAPPING_BASE) + "name").c_str(), texType, texSlot); - mat->AddProperty(&id, (std::string(_AI_MATKEY_MAPPING_BASE) + "id").c_str(), texType, texSlot); + mat->AddProperty(&name, AI_MATKEY_GLTF_MAPPINGNAME(texType, texSlot)); + mat->AddProperty(&id, AI_MATKEY_GLTF_MAPPINGID(texType, texSlot)); mat->AddProperty(&sampler->wrapS, 1, AI_MATKEY_MAPPINGMODE_U(texType, texSlot)); mat->AddProperty(&sampler->wrapT, 1, AI_MATKEY_MAPPINGMODE_V(texType, texSlot)); if (sampler->magFilter != SamplerMagFilter::UNSET) { - mat->AddProperty(&sampler->magFilter, 1, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermag").c_str(), texType, texSlot); + mat->AddProperty(&sampler->magFilter, 1, AI_MATKEY_GLTF_MAPPINGFILTER_MAG(texType, texSlot)); } if (sampler->minFilter != SamplerMinFilter::UNSET) { - mat->AddProperty(&sampler->minFilter, 1, (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermin").c_str(), texType, texSlot); + mat->AddProperty(&sampler->minFilter, 1, AI_MATKEY_GLTF_MAPPINGFILTER_MIN(texType, texSlot)); } } } @@ -235,9 +233,9 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) SetMaterialColorProperty(r, mat.pbrMetallicRoughness.baseColorFactor, aimat, AI_MATKEY_COLOR_DIFFUSE); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrMetallicRoughness.baseColorTexture, aimat, aiTextureType_DIFFUSE); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrMetallicRoughness.metallicRoughnessTexture, aimat, aiTextureType_UNKNOWN); - aimat->AddProperty(&mat.pbrMetallicRoughness.metallicFactor, 1, "$mat.gltf.pbrMetallicRoughness.metallicFactor"); - aimat->AddProperty(&mat.pbrMetallicRoughness.roughnessFactor, 1, "$mat.gltf.pbrMetallicRoughness.roughnessFactor"); + SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrMetallicRoughness.metallicRoughnessTexture, aimat, AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLICROUGHNESS_TEXTURE); + aimat->AddProperty(&mat.pbrMetallicRoughness.metallicFactor, 1, AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR); + aimat->AddProperty(&mat.pbrMetallicRoughness.roughnessFactor, 1, AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.normalTexture, aimat, aiTextureType_NORMALS); SetMaterialTextureProperty(embeddedTexIdxs, r, mat.occlusionTexture, aimat, aiTextureType_LIGHTMAP); @@ -245,17 +243,19 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) SetMaterialColorProperty(r, mat.emissiveFactor, aimat, AI_MATKEY_COLOR_EMISSIVE); aimat->AddProperty(&mat.doubleSided, 1, AI_MATKEY_TWOSIDED); - aimat->AddProperty(&mat.alphaMode, 1, "$mat.gltf.alphaMode"); - aimat->AddProperty(&mat.alphaCutoff, 1, "$mat.gltf.alphaCutoff"); + aimat->AddProperty(&mat.alphaMode, 1, AI_MATKEY_GLTF_ALPHAMODE); + aimat->AddProperty(&mat.alphaCutoff, 1, AI_MATKEY_GLTF_ALPHACUTOFF); //pbrSpecularGlossiness if (mat.pbrSpecularGlossiness.isPresent) { - aimat->AddProperty(&mat.pbrSpecularGlossiness.isPresent, 1, "$mat.gltf.pbrSpecularGlossiness.on"); - SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.value.diffuseFactor, aimat, "$clr.diffuse", 0, 1); - SetMaterialColorProperty(r, mat.pbrSpecularGlossiness.value.specularFactor, aimat, "$clr.specular", 0, 1); - aimat->AddProperty(&mat.pbrSpecularGlossiness.value.glossinessFactor, 1, "$mat.gltf.pbrSpecularGlossiness.glossinessFactor"); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.value.diffuseTexture, aimat, aiTextureType_DIFFUSE, 1); - SetMaterialTextureProperty(embeddedTexIdxs, r, mat.pbrSpecularGlossiness.value.specularGlossinessTexture, aimat, aiTextureType_UNKNOWN, 1); + PbrSpecularGlossiness &pbrSG = mat.pbrSpecularGlossiness.value; + + aimat->AddProperty(&mat.pbrSpecularGlossiness.isPresent, 1, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS); + SetMaterialColorProperty(r, pbrSG.diffuseFactor, aimat, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_FACTOR); + SetMaterialColorProperty(r, pbrSG.specularFactor, aimat, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULAR_FACTOR); + aimat->AddProperty(&pbrSG.glossinessFactor, 1, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_GLOSSINESS_FACTOR); + SetMaterialTextureProperty(embeddedTexIdxs, r, pbrSG.diffuseTexture, aimat, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_TEXTURE); + SetMaterialTextureProperty(embeddedTexIdxs, r, pbrSG.specularGlossinessTexture, aimat, AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULARGLOSSINESS_TEXTURE); } } } From da6a252efbb433e25d61e0dbdae9873d3829da76 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 5 Sep 2017 18:06:59 -0400 Subject: [PATCH 121/305] Fix METALLIC_FACTOR typo --- code/glTF2Asset.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index b4369fe71..d894e9473 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -166,7 +166,7 @@ namespace glTF2 #define AI_GLB_MAGIC_NUMBER "glTF" #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0,0 - #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0, 0 + #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR "$mat.gltf.pbrMetallicRoughness.roughnessFactor", 0, 0 #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLICROUGHNESS_TEXTURE aiTextureType_UNKNOWN, 0 #define AI_MATKEY_GLTF_ALPHAMODE "$mat.gltf.alphaMode", 0, 0 #define AI_MATKEY_GLTF_ALPHACUTOFF "$mat.gltf.alphaCutoff", 0, 0 From 37582131f494babbf3ad4ba878796fb0384dc085 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 6 Sep 2017 14:16:52 -0400 Subject: [PATCH 122/305] =?UTF-8?q?Set=20the=20metallicFactor=20to=200=20i?= =?UTF-8?q?f=20source=20file=20doesn=E2=80=99t=20have=20metallicFactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise, the models have full metallic materials and appear very dim --- code/glTF2Exporter.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 931e491e3..a5c74fd46 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -418,7 +418,12 @@ void glTF2Exporter::ExportMaterials() GetMatTex(mat, m->pbrMetallicRoughness.baseColorTexture, aiTextureType_DIFFUSE); GetMatTex(mat, m->pbrMetallicRoughness.metallicRoughnessTexture, AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLICROUGHNESS_TEXTURE); GetMatColor(mat, m->pbrMetallicRoughness.baseColorFactor, AI_MATKEY_COLOR_DIFFUSE); - mat->Get(AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR, m->pbrMetallicRoughness.metallicFactor); + + if (mat->Get(AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR, m->pbrMetallicRoughness.metallicFactor) != AI_SUCCESS) { + //if metallicFactor wasn't defined, then the source is likely not a PBR file, and the metallicFactor should be 0 + m->pbrMetallicRoughness.metallicFactor = 0; + } + mat->Get(AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR, m->pbrMetallicRoughness.roughnessFactor); GetMatTex(mat, m->normalTexture, aiTextureType_NORMALS); From 0a8183531e0fc43780e51008c04bb2d34ecb5eb5 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 6 Sep 2017 14:17:24 -0400 Subject: [PATCH 123/305] =?UTF-8?q?Set=20alphaMode,=20baseColorFactor=20op?= =?UTF-8?q?acity=20when=20model=E2=80=99s=20opacity=20isn=E2=80=99t=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- code/glTF2Exporter.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index a5c74fd46..9130c750c 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -433,7 +433,17 @@ void glTF2Exporter::ExportMaterials() mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); mat->Get(AI_MATKEY_GLTF_ALPHACUTOFF, m->alphaCutoff); - mat->Get(AI_MATKEY_GLTF_ALPHAMODE, m->alphaMode); + + if (mat->Get(AI_MATKEY_GLTF_ALPHAMODE, m->alphaMode) != AI_SUCCESS) { + float opacity; + + if (mat->Get(AI_MATKEY_OPACITY, opacity) == AI_SUCCESS) { + if (opacity < 1) { + m->alphaMode = "MASK"; + m->pbrMetallicRoughness.baseColorFactor[3] *= opacity; + } + } + } bool hasPbrSpecularGlossiness; mat->Get(AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS, hasPbrSpecularGlossiness); From a5e8e0b2bd8fbdd625920ccf448d9475f327fed5 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 6 Sep 2017 14:17:37 -0400 Subject: [PATCH 124/305] Remove commented out code --- code/glTF2Asset.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index d894e9473..12cad25ba 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -717,13 +717,6 @@ namespace glTF2 float strength = 1; }; - //! Holds a material property that can be a texture or a color (fallback for glTF 1) - /*struct FallbackTexProperty - { - Ref texture; - vec4 color; - };*/ - struct PbrMetallicRoughness { vec4 baseColorFactor; From 2ee7991558be791115087670026b9039cfac6bff Mon Sep 17 00:00:00 2001 From: John Senneker Date: Wed, 6 Sep 2017 14:49:25 -0400 Subject: [PATCH 125/305] Restrict search for OFF header to first 3 bytes --- code/OFFLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/OFFLoader.cpp b/code/OFFLoader.cpp index 2723beb0e..5038176e2 100644 --- a/code/OFFLoader.cpp +++ b/code/OFFLoader.cpp @@ -94,7 +94,7 @@ bool OFFImporter::CanRead( const std::string& pFile, IOSystem* pIOHandler, bool { if (!pIOHandler)return true; const char* tokens[] = {"off"}; - return SearchFileHeaderForToken(pIOHandler,pFile,tokens,1); + return SearchFileHeaderForToken(pIOHandler,pFile,tokens,1,3); } return false; } From 19876e98227ebf3a76ad994cd1f190c1b382ca82 Mon Sep 17 00:00:00 2001 From: John Senneker Date: Wed, 6 Sep 2017 14:50:02 -0400 Subject: [PATCH 126/305] Add support for importing both glTF and glTF2 files --- code/ImporterRegistry.cpp | 2 ++ code/glTF2Importer.cpp | 28 ++++++++++++---------------- code/glTFAsset.inl | 4 ---- code/glTFImporter.cpp | 26 ++++++++++---------------- 4 files changed, 24 insertions(+), 36 deletions(-) diff --git a/code/ImporterRegistry.cpp b/code/ImporterRegistry.cpp index 2b5d9c1a0..b4d2c3dcf 100644 --- a/code/ImporterRegistry.cpp +++ b/code/ImporterRegistry.cpp @@ -181,6 +181,7 @@ corresponding preprocessor flag to selectively disable formats. # include "AssbinLoader.h" #endif #ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER +# include "glTFImporter.h" # include "glTF2Importer.h" #endif #ifndef ASSIMP_BUILD_NO_C4D_IMPORTER @@ -335,6 +336,7 @@ void GetImporterInstanceList(std::vector< BaseImporter* >& out) out.push_back( new AssbinImporter() ); #endif #if ( !defined ASSIMP_BUILD_NO_GLTF_IMPORTER ) + out.push_back( new glTFImporter() ); out.push_back( new glTF2Importer() ); #endif #if ( !defined ASSIMP_BUILD_NO_C4D_IMPORTER ) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 860974243..1ab23fd18 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -58,6 +58,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "glTF2Asset.h" // This is included here so WriteLazyDict's definition is found. #include "glTF2AssetWriter.h" +#include +#include using namespace Assimp; using namespace glTF2; @@ -100,24 +102,18 @@ const aiImporterDesc* glTF2Importer::GetInfo() const bool glTF2Importer::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool checkSig) const { - const std::string& extension = GetExtension(pFile); + const std::string &extension = GetExtension(pFile); - if (extension == "gltf" || extension == "glb") - return true; + if (extension != "gltf") // We currently can't read glTF2 binary files (.glb) + return false; - if ((checkSig || !extension.length()) && pIOHandler) { - char buffer[4]; - - std::unique_ptr pStream(pIOHandler->Open(pFile)); - if (pStream && pStream->Read(buffer, sizeof(buffer), 1) == 1) { - if (memcmp(buffer, AI_GLB_MAGIC_NUMBER, sizeof(buffer)) == 0) { - return true; // Has GLB header - } - else if (memcmp(buffer, "{\r\n ", sizeof(buffer)) == 0 - || memcmp(buffer, "{\n ", sizeof(buffer)) == 0) { - // seems a JSON file, and we're the only format that can read them - return true; - } + if (checkSig && pIOHandler) { + glTF2::Asset asset(pIOHandler); + try { + asset.Load(pFile, extension == "glb"); + return asset.asset.version >= 2; + } catch (...) { + return false; } } diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index 91d36c59b..32fe77288 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -1243,10 +1243,6 @@ inline void AssetMetadata::Read(Document& doc) } version = std::max(statedVersion, version); - if (version == 0) { - // if missing version, we'll assume version 1... - version = 1; - } if (version != 1) { char msg[128]; diff --git a/code/glTFImporter.cpp b/code/glTFImporter.cpp index 0ded26294..6dbeb7e91 100644 --- a/code/glTFImporter.cpp +++ b/code/glTFImporter.cpp @@ -100,24 +100,18 @@ const aiImporterDesc* glTFImporter::GetInfo() const bool glTFImporter::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool checkSig) const { - const std::string& extension = GetExtension(pFile); + const std::string &extension = GetExtension(pFile); - if (extension == "gltf" || extension == "glb") - return true; + if (extension != "gltf" && extension != "glb") + return false; - if ((checkSig || !extension.length()) && pIOHandler) { - char buffer[4]; - - std::unique_ptr pStream(pIOHandler->Open(pFile)); - if (pStream && pStream->Read(buffer, sizeof(buffer), 1) == 1) { - if (memcmp(buffer, AI_GLB_MAGIC_NUMBER, sizeof(buffer)) == 0) { - return true; // Has GLB header - } - else if (memcmp(buffer, "{\r\n ", sizeof(buffer)) == 0 - || memcmp(buffer, "{\n ", sizeof(buffer)) == 0) { - // seems a JSON file, and we're the only format that can read them - return true; - } + if (checkSig && pIOHandler) { + glTF::Asset asset(pIOHandler); + try { + asset.Load(pFile, extension == "glb"); + return asset.asset.version < 2; + } catch (...) { + return false; } } From 140b903d7a062216016597ef30f4885b28e5c767 Mon Sep 17 00:00:00 2001 From: John Senneker Date: Wed, 6 Sep 2017 15:32:44 -0400 Subject: [PATCH 127/305] Fix parsing of glTF version Handle version as int in gltf Fix format specifiers in glTF version parser --- code/glTF2Asset.h | 4 ++-- code/glTF2Asset.inl | 29 ++++++++++++----------------- code/glTF2AssetWriter.inl | 9 ++------- code/glTF2Exporter.cpp | 2 +- code/glTF2Importer.cpp | 3 ++- code/glTFAsset.h | 4 ++-- code/glTFAsset.inl | 29 ++++++++++++++++++++--------- code/glTFAssetWriter.inl | 9 ++------- code/glTFImporter.cpp | 3 ++- 9 files changed, 45 insertions(+), 47 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 12cad25ba..1711b930b 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -1005,13 +1005,13 @@ namespace glTF2 std::string version; //!< Specifies the target rendering API (default: "1.0.3") } profile; //!< Specifies the target rendering API and version, e.g., WebGL 1.0.3. (default: {}) - float version; //!< The glTF format version + std::string version; //!< The glTF format version void Read(Document& doc); AssetMetadata() : premultipliedAlpha(false) - , version(0) + , version("") { } }; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index e2d812563..bc8261317 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -985,15 +985,21 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { - // read the version, etc. - std::string statedVersion; - if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); premultipliedAlpha = MemberOrDefault(*obj, "premultipliedAlpha", false); - statedVersion = MemberOrDefault(*obj, "version", "0.0"); + + if (Value* versionString = FindString(*obj, "version")) { + version = versionString->GetString(); + } else if (Value* versionNumber = FindNumber (*obj, "version")) { + char buf[4]; + + ai_snprintf(buf, 4, "%.1f", versionNumber->GetDouble()); + + version = buf; + } if (Value* profile = FindObject(*obj, "profile")) { ReadMember(*profile, "api", this->profile.api); @@ -1001,19 +1007,8 @@ inline void AssetMetadata::Read(Document& doc) } } - float statedFloatVersion = std::strtof(statedVersion.c_str(), 0); - - version = std::max(statedFloatVersion, version); - - if (version == 0) { - // if missing version, we'll assume version 2.0... - version = 2; - } - - if (version != 2) { - char msg[128]; - ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.1f", version); - throw DeadlyImportError(msg); + if (version.empty() || version[0] != '2') { + throw DeadlyImportError("GLTF: Unsupported glTF version: " + version); } } diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index ad68a14e1..1665bee02 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -652,13 +652,8 @@ namespace glTF2 { { Value asset; asset.SetObject(); - { - char versionChar[10]; - ai_snprintf(versionChar, sizeof(versionChar), "%.1f", mAsset.asset.version); - asset.AddMember("version", Value(versionChar, mAl).Move(), mAl); - - asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); - } + asset.AddMember("version", Value(mAsset.asset.version, mAl).Move(), mAl); + asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); mDoc.AddMember("asset", asset, mAl); } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 9130c750c..f3e083543 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -811,7 +811,7 @@ void glTF2Exporter::ExportScene() void glTF2Exporter::ExportMetadata() { AssetMetadata& asset = mAsset->asset; - asset.version = 2; + asset.version = "2.0"; char buffer[256]; ai_snprintf(buffer, 256, "Open Asset Import Library (assimp v%d.%d.%d)", diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 1ab23fd18..34144b9e2 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -111,7 +111,8 @@ bool glTF2Importer::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool glTF2::Asset asset(pIOHandler); try { asset.Load(pFile, extension == "glb"); - return asset.asset.version >= 2; + std::string version = asset.asset.version; + return !version.empty() && version[0] == '2'; } catch (...) { return false; } diff --git a/code/glTFAsset.h b/code/glTFAsset.h index 30d0bc056..60107bee6 100644 --- a/code/glTFAsset.h +++ b/code/glTFAsset.h @@ -1058,13 +1058,13 @@ namespace glTF std::string version; //!< Specifies the target rendering API (default: "1.0.3") } profile; //!< Specifies the target rendering API and version, e.g., WebGL 1.0.3. (default: {}) - float version; //!< The glTF format version (should be 1.0) + std::string version; //!< The glTF format version (should be 1.0) void Read(Document& doc); AssetMetadata() : premultipliedAlpha(false) - , version(0) + , version("") { } }; diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index 32fe77288..bb4bb50aa 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "StringUtils.h" +#include // Header files, Assimp #include @@ -128,6 +129,12 @@ namespace { return (it != val.MemberEnd() && it->value.IsString()) ? &it->value : 0; } + inline Value* FindNumber(Value& val, const char* id) + { + Value::MemberIterator it = val.FindMember(id); + return (it != val.MemberEnd() && it->value.IsNumber()) ? &it->value : 0; + } + inline Value* FindArray(Value& val, const char* id) { Value::MemberIterator it = val.FindMember(id); @@ -1228,13 +1235,21 @@ inline void Scene::Read(Value& obj, Asset& r) inline void AssetMetadata::Read(Document& doc) { // read the version, etc. - float statedVersion = 0; if (Value* obj = FindObject(doc, "asset")) { ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); premultipliedAlpha = MemberOrDefault(*obj, "premultipliedAlpha", false); - statedVersion = MemberOrDefault(*obj, "version", 0); + + if (Value* versionString = FindString(*obj, "version")) { + version = versionString->GetString(); + } else if (Value* versionNumber = FindNumber (*obj, "version")) { + char buf[4]; + + ai_snprintf(buf, 4, "%.1f", versionNumber->GetDouble()); + + version = buf; + } if (Value* profile = FindObject(*obj, "profile")) { ReadMember(*profile, "api", this->profile.api); @@ -1242,12 +1257,8 @@ inline void AssetMetadata::Read(Document& doc) } } - version = std::max(statedVersion, version); - - if (version != 1) { - char msg[128]; - ai_snprintf(msg, 128, "GLTF: Unsupported glTF version: %.0f", version); - throw DeadlyImportError(msg); + if (version.empty() || version[0] != '1') { + throw DeadlyImportError("GLTF: Unsupported glTF version: " + version); } } @@ -1269,7 +1280,7 @@ inline void Asset::ReadBinaryHeader(IOStream& stream) } AI_SWAP4(header.version); - asset.version = header.version; + asset.version = std::to_string(header.version); if (header.version != 1) { throw DeadlyImportError("GLTF: Unsupported binary glTF version"); } diff --git a/code/glTFAssetWriter.inl b/code/glTFAssetWriter.inl index 166b84fd1..698f0ba8f 100644 --- a/code/glTFAssetWriter.inl +++ b/code/glTFAssetWriter.inl @@ -606,13 +606,8 @@ namespace glTF { { Value asset; asset.SetObject(); - { - char versionChar[10]; - ai_snprintf(versionChar, sizeof(versionChar), "%.0f", mAsset.asset.version); - asset.AddMember("version", Value(versionChar, mAl).Move(), mAl); - - asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); - } + asset.AddMember("version", Value(mAsset.asset.version, mAl).Move(), mAl); + asset.AddMember("generator", Value(mAsset.asset.generator, mAl).Move(), mAl); mDoc.AddMember("asset", asset, mAl); } diff --git a/code/glTFImporter.cpp b/code/glTFImporter.cpp index 6dbeb7e91..1b5c953be 100644 --- a/code/glTFImporter.cpp +++ b/code/glTFImporter.cpp @@ -109,7 +109,8 @@ bool glTFImporter::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool glTF::Asset asset(pIOHandler); try { asset.Load(pFile, extension == "glb"); - return asset.asset.version < 2; + std::string version = asset.asset.version; + return !version.empty() && version[0] == '1'; } catch (...) { return false; } From a438ece655ced52835bda0dcb492cab8394de14f Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 6 Sep 2017 23:43:43 -0400 Subject: [PATCH 128/305] Remove premultipliedAlpha from gltf2 --- code/glTF2Asset.h | 7 +------ code/glTF2Asset.inl | 2 -- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 1711b930b..8b226dacb 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -998,7 +998,6 @@ namespace glTF2 { std::string copyright; //!< A copyright message suitable for display to credit the content creator. std::string generator; //!< Tool that generated this glTF model.Useful for debugging. - bool premultipliedAlpha; //!< Specifies if the shaders were generated with premultiplied alpha. (default: false) struct { std::string api; //!< Specifies the target rendering API (default: "WebGL") @@ -1009,11 +1008,7 @@ namespace glTF2 void Read(Document& doc); - AssetMetadata() - : premultipliedAlpha(false) - , version("") - { - } + AssetMetadata() : version("") {} }; // diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index bc8261317..3d4a4ee42 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -989,8 +989,6 @@ inline void AssetMetadata::Read(Document& doc) ReadMember(*obj, "copyright", copyright); ReadMember(*obj, "generator", generator); - premultipliedAlpha = MemberOrDefault(*obj, "premultipliedAlpha", false); - if (Value* versionString = FindString(*obj, "version")) { version = versionString->GetString(); } else if (Value* versionNumber = FindNumber (*obj, "version")) { From 5cb13aa4b3b4dc282ed970ffd57238b823492f47 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 7 Sep 2017 14:55:44 -0400 Subject: [PATCH 129/305] Load gltf .bin files from correct directory --- code/glTF2Asset.inl | 4 +++- code/glTFAsset.inl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 3d4a4ee42..03c2279e8 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -337,7 +337,9 @@ inline void Buffer::Read(Value& obj, Asset& r) } else { // Local file if (byteLength > 0) { - IOStream* file = r.OpenFile(uri, "rb"); + std::string dir = !r.mCurrentAssetDir.empty() ? (r.mCurrentAssetDir + "/") : ""; + + IOStream* file = r.OpenFile(dir + uri, "rb"); if (file) { bool ok = LoadFromStream(*file, byteLength); delete file; diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index bb4bb50aa..9284ccb02 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -316,7 +316,9 @@ inline void Buffer::Read(Value& obj, Asset& r) } else { // Local file if (byteLength > 0) { - IOStream* file = r.OpenFile(uri, "rb"); + std::string dir = !r.mCurrentAssetDir.empty() ? (r.mCurrentAssetDir + "/") : ""; + + IOStream* file = r.OpenFile(dir + uri, "rb"); if (file) { bool ok = LoadFromStream(*file, byteLength); delete file; From d518289e7217f1966b8abd33bd574943781fab31 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 7 Sep 2017 15:48:11 -0400 Subject: [PATCH 130/305] more specific token search for Collada Loader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit collada can appear in many files, such as glTFs via the “generator” field (in the form of collada2gltf) --- code/ColladaLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ColladaLoader.cpp b/code/ColladaLoader.cpp index f7ef903f4..0e970b468 100644 --- a/code/ColladaLoader.cpp +++ b/code/ColladaLoader.cpp @@ -119,7 +119,7 @@ bool ColladaLoader::CanRead( const std::string& pFile, IOSystem* pIOHandler, boo * might be NULL and it's our duty to return true here. */ if (!pIOHandler)return true; - const char* tokens[] = {"collada"}; + const char* tokens[] = {" Date: Thu, 7 Sep 2017 15:50:35 -0400 Subject: [PATCH 131/305] Add gltf2 basic unit test --- test/CMakeLists.txt | 5 ++- test/unit/utglTF2ImportExport.cpp | 75 +++++++++++++++++++++++++++++++ test/unit/utglTFImportExport.cpp | 2 +- 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 test/unit/utglTF2ImportExport.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cb559deb3..3520f9782 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -65,7 +65,7 @@ SET( TEST_SRCS unit/ut3DSImportExport.cpp unit/utACImportExport.cpp unit/utAMFImportExport.cpp - unit/utASEImportExport.cpp + unit/utASEImportExport.cpp unit/utAnim.cpp unit/AssimpAPITest.cpp unit/utB3DImportExport.cpp @@ -88,6 +88,7 @@ SET( TEST_SRCS unit/utFixInfacingNormals.cpp unit/utGenNormals.cpp unit/utglTFImportExport.cpp + unit/utglTF2ImportExport.cpp unit/utHMPImportExport.cpp unit/utIFCImportExport.cpp unit/utImporter.cpp @@ -146,7 +147,7 @@ add_executable( unit ) add_definitions(-DASSIMP_TEST_MODELS_DIR="${CMAKE_CURRENT_LIST_DIR}/models") - + SET_PROPERTY( TARGET assimp PROPERTY DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) IF( WIN32 ) diff --git a/test/unit/utglTF2ImportExport.cpp b/test/unit/utglTF2ImportExport.cpp new file mode 100644 index 000000000..09c43131d --- /dev/null +++ b/test/unit/utglTF2ImportExport.cpp @@ -0,0 +1,75 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#include "UnitTestPCH.h" +#include "AbstractImportExportBase.h" + +#include +#include + +using namespace Assimp; + +class utglTF2ImportExport : public AbstractImportExportBase { +public: + virtual bool importerTest() { + Assimp::Importer importer; + const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured.gltf", 0); + return nullptr != scene; + } + + virtual bool exporterTest() { + Assimp::Importer importer; + Assimp::Exporter exporter; + const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured.gltf", 0 ); + EXPECT_NE( nullptr, scene ); + EXPECT_EQ( aiReturn_SUCCESS, exporter.Export( scene, "gltf2", ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured_out.gltf" ) ); + + return true; + } +}; + +TEST_F( utglTF2ImportExport, importglTF2FromFileTest ) { + EXPECT_TRUE( importerTest() ); +} + +TEST_F( utglTF2ImportExport, exportglTF2FromFileTest ) { + EXPECT_TRUE( exporterTest() ); +} diff --git a/test/unit/utglTFImportExport.cpp b/test/unit/utglTFImportExport.cpp index 349dddafa..18c348299 100644 --- a/test/unit/utglTFImportExport.cpp +++ b/test/unit/utglTFImportExport.cpp @@ -55,6 +55,6 @@ public: } }; -TEST_F( utglTFImportExport, importglTFromFileTest ) { +TEST_F( utglTFImportExport, importglTFFromFileTest ) { EXPECT_TRUE( importerTest() ); } From b4f5033d899f443b55d756a65c6369204dfffae1 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 7 Sep 2017 17:15:05 -0400 Subject: [PATCH 132/305] Remove compresssed file format flag --- code/glTF2Importer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 34144b9e2..d234afab3 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -74,8 +74,7 @@ static const aiImporterDesc desc = { "", "", "", - aiImporterFlags_SupportTextFlavour | aiImporterFlags_SupportBinaryFlavour | aiImporterFlags_SupportCompressedFlavour - | aiImporterFlags_LimitedSupport | aiImporterFlags_Experimental, + aiImporterFlags_SupportTextFlavour | aiImporterFlags_SupportBinaryFlavour | aiImporterFlags_LimitedSupport | aiImporterFlags_Experimental, 0, 0, 0, From 816e6909ca87dd8f77a8375d3fa38de004371556 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Thu, 7 Sep 2017 17:32:41 -0400 Subject: [PATCH 133/305] Remove KHR_binary_glTF code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Binary glTF is now part of the glTF2 spec. However, it’s implemented incorrectly, so will be temporarily removed --- code/glTF2Asset.h | 21 +-------- code/glTF2Asset.inl | 84 ++---------------------------------- code/glTF2AssetWriter.h | 2 - code/glTF2AssetWriter.inl | 90 +-------------------------------------- code/glTF2Exporter.cpp | 10 +---- code/glTF2Importer.cpp | 11 ++--- 6 files changed, 10 insertions(+), 208 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 8b226dacb..b093ec027 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -43,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Declares a glTF class to handle gltf/glb files * * glTF Extensions Support: - * KHR_binary_glTF: full * KHR_materials_pbrSpecularGlossiness full */ #ifndef GLTF2ASSET_H_INC @@ -193,17 +192,6 @@ namespace glTF2 #include "./../include/assimp/Compiler/pushpack1.h" #endif - //! For the KHR_binary_glTF extension (binary .glb file) - //! 20-byte header (+ the JSON + a "body" data section) - struct GLB_Header - { - uint8_t magic[4]; //!< Magic number: "glTF" - uint32_t version; //!< Version number - uint32_t length; //!< Total length of the Binary glTF, including header, scene, and body, in bytes - uint32_t sceneLength; //!< Length, in bytes, of the glTF scene - uint32_t sceneFormat; //!< Specifies the format of the glTF scene (see the SceneFormat enum) - } PACK_STRUCT; - #ifdef ASSIMP_API #include "./../include/assimp/Compiler/poppack1.h" #endif @@ -1049,7 +1037,6 @@ namespace glTF2 //! Keeps info about the enabled extensions struct Extensions { - bool KHR_binary_glTF; bool KHR_materials_pbrSpecularGlossiness; } extensionsUsed; @@ -1097,10 +1084,7 @@ namespace glTF2 } //! Main function - void Load(const std::string& file, bool isBinary = false); - - //! Enables the "KHR_binary_glTF" extension on the asset - void SetAsBinary(); + void Load(const std::string& file); //! Search for an available name, starting from the given strings std::string FindUniqueID(const std::string& str, const char* suffix); @@ -1109,11 +1093,8 @@ namespace glTF2 { return mBodyBuffer; } private: - void ReadBinaryHeader(IOStream& stream); - void ReadExtensionsUsed(Document& doc); - IOStream* OpenFile(std::string path, const char* mode, bool absolute = false); }; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 03c2279e8..cbdac5d80 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -290,11 +290,6 @@ inline Buffer::~Buffer() inline const char* Buffer::TranslateId(Asset& r, const char* id) { - // Compatibility with old spec - if (r.extensionsUsed.KHR_binary_glTF && strcmp(id, "KHR_binary_glTF") == 0) { - return "binary_glTF"; - } - return id; } @@ -630,28 +625,6 @@ inline Image::Image() inline void Image::Read(Value& obj, Asset& r) { - // Check for extensions first (to detect binary embedded data) - if (Value* extensions = FindObject(obj, "extensions")) { - if (r.extensionsUsed.KHR_binary_glTF) { - if (Value* ext = FindObject(*extensions, "KHR_binary_glTF")) { - - width = MemberOrDefault(*ext, "width", 0); - height = MemberOrDefault(*ext, "height", 0); - - ReadMember(*ext, "mimeType", mimeType); - - if (Value* bufferViewVal = FindUInt(*ext, "bufferView")) { - Ref bv = r.bufferViews.Retrieve(bufferViewVal->GetUint()); - if (bv) { - mDataLength = bv->byteLength; - mData = new uint8_t[mDataLength]; - memcpy(mData, bv->buffer->GetPointer() + bv->byteOffset, mDataLength); - } - } - } - } - } - if (!mDataLength) { if (Value* uri = FindString(obj, "uri")) { const char* uristr = uri->GetString(); @@ -1016,40 +989,7 @@ inline void AssetMetadata::Read(Document& doc) // Asset methods implementation // -inline void Asset::ReadBinaryHeader(IOStream& stream) -{ - GLB_Header header; - if (stream.Read(&header, sizeof(header), 1) != 1) { - throw DeadlyImportError("GLTF: Unable to read the file header"); - } - - if (strncmp((char*)header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)) != 0) { - throw DeadlyImportError("GLTF: Invalid binary glTF file"); - } - - AI_SWAP4(header.version); - asset.version = header.version; - if (header.version != 1) { - throw DeadlyImportError("GLTF: Unsupported binary glTF version"); - } - - AI_SWAP4(header.sceneFormat); - if (header.sceneFormat != SceneFormat_JSON) { - throw DeadlyImportError("GLTF: Unsupported binary glTF scene format"); - } - - AI_SWAP4(header.length); - AI_SWAP4(header.sceneLength); - - mSceneLength = static_cast(header.sceneLength); - - mBodyOffset = sizeof(header)+mSceneLength; - mBodyOffset = (mBodyOffset + 3) & ~3; // Round up to next multiple of 4 - - mBodyLength = header.length - mBodyOffset; -} - -inline void Asset::Load(const std::string& pFile, bool isBinary) +inline void Asset::Load(const std::string& pFile) { mCurrentAssetDir.clear(); int pos = std::max(int(pFile.rfind('/')), int(pFile.rfind('\\'))); @@ -1060,16 +1000,8 @@ inline void Asset::Load(const std::string& pFile, bool isBinary) throw DeadlyImportError("GLTF: Could not open file for reading"); } - // is binary? then read the header - if (isBinary) { - SetAsBinary(); // also creates the body buffer - ReadBinaryHeader(*stream); - } - else { - mSceneLength = stream->FileSize(); - mBodyLength = 0; - } - + mSceneLength = stream->FileSize(); + mBodyLength = 0; // read the scene data @@ -1130,15 +1062,6 @@ inline void Asset::Load(const std::string& pFile, bool isBinary) } } -inline void Asset::SetAsBinary() -{ - if (!extensionsUsed.KHR_binary_glTF) { - extensionsUsed.KHR_binary_glTF = true; - mBodyBuffer = buffers.Create("binary_glTF"); - mBodyBuffer->MarkAsSpecial(); - } -} - inline void Asset::ReadExtensionsUsed(Document& doc) { Value* extsUsed = FindArray(doc, "extensionsUsed"); @@ -1155,7 +1078,6 @@ inline void Asset::ReadExtensionsUsed(Document& doc) #define CHECK_EXT(EXT) \ if (exts.find(#EXT) != exts.end()) extensionsUsed.EXT = true; - CHECK_EXT(KHR_binary_glTF); CHECK_EXT(KHR_materials_pbrSpecularGlossiness); #undef CHECK_EXT diff --git a/code/glTF2AssetWriter.h b/code/glTF2AssetWriter.h index 97efbeab5..bce2b1bd1 100644 --- a/code/glTF2AssetWriter.h +++ b/code/glTF2AssetWriter.h @@ -43,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Declares a class to write gltf/glb files * * glTF Extensions Support: - * KHR_binary_glTF: full * KHR_materials_pbrSpecularGlossiness: full */ #ifndef GLTF2ASSETWRITER_H_INC @@ -82,7 +81,6 @@ public: AssetWriter(Asset& asset); void WriteFile(const char* path); - void WriteGLBFile(const char* path); }; } diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 1665bee02..e62f2377e 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -179,21 +179,7 @@ namespace glTF2 { inline void Write(Value& obj, Image& img, AssetWriter& w) { std::string uri; - if (w.mAsset.extensionsUsed.KHR_binary_glTF && img.bufferView) { - Value exts, ext; - exts.SetObject(); - ext.SetObject(); - - ext.AddMember("bufferView", img.bufferView->index, w.mAl); - - if (!img.mimeType.empty()) - ext.AddMember("mimeType", StringRef(img.mimeType), w.mAl); - - exts.AddMember("KHR_binary_glTF", ext, w.mAl); - obj.AddMember("extensions", exts, w.mAl); - return; - } - else if (img.HasData()) { + if (img.HasData()) { uri = "data:" + (img.mimeType.empty() ? "application/octet-stream" : img.mimeType); uri += ";base64,"; Util::EncodeBase64(img.GetData(), img.GetDataLength(), uri); @@ -577,77 +563,6 @@ namespace glTF2 { } } - inline void AssetWriter::WriteGLBFile(const char* path) - { - std::unique_ptr outfile(mAsset.OpenFile(path, "wb", true)); - - if (outfile == 0) { - throw DeadlyExportError("Could not open output file: " + std::string(path)); - } - - // we will write the header later, skip its size - outfile->Seek(sizeof(GLB_Header), aiOrigin_SET); - - StringBuffer docBuffer; - Writer writer(docBuffer); - mDoc.Accept(writer); - - if (outfile->Write(docBuffer.GetString(), docBuffer.GetSize(), 1) != 1) { - throw DeadlyExportError("Failed to write scene data!"); - } - - WriteBinaryData(outfile.get(), docBuffer.GetSize()); - } - - inline void AssetWriter::WriteBinaryData(IOStream* outfile, size_t sceneLength) - { - // - // write the body data - // - - size_t bodyLength = 0; - if (Ref b = mAsset.GetBodyBuffer()) { - bodyLength = b->byteLength; - - if (bodyLength > 0) { - size_t bodyOffset = sizeof(GLB_Header) + sceneLength; - bodyOffset = (bodyOffset + 3) & ~3; // Round up to next multiple of 4 - - outfile->Seek(bodyOffset, aiOrigin_SET); - - if (outfile->Write(b->GetPointer(), b->byteLength, 1) != 1) { - throw DeadlyExportError("Failed to write body data!"); - } - } - } - - // - // write the header - // - - GLB_Header header; - memcpy(header.magic, AI_GLB_MAGIC_NUMBER, sizeof(header.magic)); - - header.version = 2; - AI_SWAP4(header.version); - - header.length = uint32_t(sizeof(header) + sceneLength + bodyLength); - AI_SWAP4(header.length); - - header.sceneLength = uint32_t(sceneLength); - AI_SWAP4(header.sceneLength); - - header.sceneFormat = SceneFormat_JSON; - AI_SWAP4(header.sceneFormat); - - outfile->Seek(0, aiOrigin_SET); - - if (outfile->Write(&header, 1, sizeof(header)) != sizeof(header)) { - throw DeadlyExportError("Failed to write the header!"); - } - } - - inline void AssetWriter::WriteMetadata() { Value asset; @@ -662,9 +577,6 @@ namespace glTF2 { Value exts; exts.SetArray(); { - //if (false) - // exts.PushBack(StringRef("KHR_binary_glTF"), mAl); - // This is used to export pbrSpecularGlossiness materials with GLTF 2. if (this->mAsset.extensionsUsed.KHR_materials_pbrSpecularGlossiness) { exts.PushBack(StringRef("KHR_materials_pbrSpecularGlossiness"), mAl); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index f3e083543..e8c42128e 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -101,10 +101,6 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai mAsset.reset( new Asset( pIOSystem ) ); - if (isBinary) { - mAsset->SetAsBinary(); - } - ExportMetadata(); ExportMaterials(); @@ -121,11 +117,7 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai AssetWriter writer(*mAsset); - if (isBinary) { - writer.WriteGLBFile(filename); - } else { - writer.WriteFile(filename); - } + writer.WriteFile(filename); } /* diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index d234afab3..34f12b5af 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -74,7 +74,7 @@ static const aiImporterDesc desc = { "", "", "", - aiImporterFlags_SupportTextFlavour | aiImporterFlags_SupportBinaryFlavour | aiImporterFlags_LimitedSupport | aiImporterFlags_Experimental, + aiImporterFlags_SupportTextFlavour | aiImporterFlags_LimitedSupport | aiImporterFlags_Experimental, 0, 0, 0, @@ -103,13 +103,13 @@ bool glTF2Importer::CanRead(const std::string& pFile, IOSystem* pIOHandler, bool { const std::string &extension = GetExtension(pFile); - if (extension != "gltf") // We currently can't read glTF2 binary files (.glb) + if (extension != "gltf") // We currently can't read glTF2 binary files (.glb), yet return false; if (checkSig && pIOHandler) { glTF2::Asset asset(pIOHandler); try { - asset.Load(pFile, extension == "glb"); + asset.Load(pFile); std::string version = asset.asset.version; return !version.empty() && version[0] == '2'; } catch (...) { @@ -615,15 +615,12 @@ void glTF2Importer::InternReadFile(const std::string& pFile, aiScene* pScene, IO // read the asset file glTF2::Asset asset(pIOHandler); - asset.Load(pFile, GetExtension(pFile) == "glb"); - + asset.Load(pFile); // // Copy the data out // - - ImportEmbeddedTextures(asset); ImportMaterials(asset); From 990fe143a131f96cec74c841567e9b20230220a8 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 8 Sep 2017 15:54:03 -0400 Subject: [PATCH 134/305] =?UTF-8?q?Fix=20mesh=20primitive=E2=80=99s=20attr?= =?UTF-8?q?ibutes=E2=80=99=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec TEXCOORD -> TEXCOORD_0 COLOR -> COLOR_0 JOINTS -> JOINTS_0 WEIGHTS -> WEIGHTS_0 Remove JOINTMATRIX since it’s not supported (and doesn’t seem to be output, anyway) TANGENT should be added at a later date --- code/glTF2AssetWriter.inl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index e62f2377e..ed599c23e 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -380,11 +380,10 @@ namespace glTF2 { { WriteAttrs(w, attrs, p.attributes.position, "POSITION"); WriteAttrs(w, attrs, p.attributes.normal, "NORMAL"); - WriteAttrs(w, attrs, p.attributes.texcoord, "TEXCOORD", true); - WriteAttrs(w, attrs, p.attributes.color, "COLOR"); - WriteAttrs(w, attrs, p.attributes.joint, "JOINT"); - WriteAttrs(w, attrs, p.attributes.jointmatrix, "JOINTMATRIX"); - WriteAttrs(w, attrs, p.attributes.weight, "WEIGHT"); + WriteAttrs(w, attrs, p.attributes.texcoord, "TEXCOORD_0", true); + WriteAttrs(w, attrs, p.attributes.color, "COLOR_0"); + WriteAttrs(w, attrs, p.attributes.joint, "JOINTS_0"); + WriteAttrs(w, attrs, p.attributes.weight, "WEIGHTS_0"); } prim.AddMember("attributes", attrs, w.mAl); } From b1a5ca451639dbaf4852580c6842b226f3ff860c Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 8 Sep 2017 15:58:09 -0400 Subject: [PATCH 135/305] Use `forceNumber` argument of `WriteAttrs` to write correct attribute names, instead --- code/glTF2AssetWriter.inl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index ed599c23e..c91313ab4 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -380,10 +380,10 @@ namespace glTF2 { { WriteAttrs(w, attrs, p.attributes.position, "POSITION"); WriteAttrs(w, attrs, p.attributes.normal, "NORMAL"); - WriteAttrs(w, attrs, p.attributes.texcoord, "TEXCOORD_0", true); - WriteAttrs(w, attrs, p.attributes.color, "COLOR_0"); - WriteAttrs(w, attrs, p.attributes.joint, "JOINTS_0"); - WriteAttrs(w, attrs, p.attributes.weight, "WEIGHTS_0"); + WriteAttrs(w, attrs, p.attributes.texcoord, "TEXCOORD", true); + WriteAttrs(w, attrs, p.attributes.color, "COLOR", true); + WriteAttrs(w, attrs, p.attributes.joint, "JOINTS", true); + WriteAttrs(w, attrs, p.attributes.weight, "WEIGHTS", true); } prim.AddMember("attributes", attrs, w.mAl); } From cde29c937c4fdefa0a2c02f2ec1a8bd49e4121fb Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Fri, 8 Sep 2017 16:32:00 -0400 Subject: [PATCH 136/305] Formatting --- code/glTF2Exporter.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index e8c42128e..e2b46b1f7 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -389,8 +389,6 @@ void glTF2Exporter::GetMatColor(const aiMaterial* mat, vec3& prop, const char* p void glTF2Exporter::ExportMaterials() { - bool& KHR_materials_pbrSpecularGlossiness = mAsset->extensionsUsed.KHR_materials_pbrSpecularGlossiness; - aiString aiName; for (unsigned int i = 0; i < mScene->mNumMaterials; ++i) { const aiMaterial* mat = mScene->mMaterials[i]; @@ -442,8 +440,8 @@ void glTF2Exporter::ExportMaterials() if (hasPbrSpecularGlossiness) { - if (!KHR_materials_pbrSpecularGlossiness) { - KHR_materials_pbrSpecularGlossiness = true; + if (!mAsset->extensionsUsed.KHR_materials_pbrSpecularGlossiness) { + mAsset->extensionsUsed.KHR_materials_pbrSpecularGlossiness = true; } PbrSpecularGlossiness pbrSG; From 86a8a58d126bbd58ff42edbcc71e79fb825f6922 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 10 Sep 2017 21:04:28 -0400 Subject: [PATCH 137/305] Exclude glTF2 Exporter test when ASSIMP_BUILD_NO_EXPORT --- test/unit/utglTF2ImportExport.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/unit/utglTF2ImportExport.cpp b/test/unit/utglTF2ImportExport.cpp index 09c43131d..fa69d648a 100644 --- a/test/unit/utglTF2ImportExport.cpp +++ b/test/unit/utglTF2ImportExport.cpp @@ -55,6 +55,7 @@ public: return nullptr != scene; } +#ifndef ASSIMP_BUILD_NO_EXPORT virtual bool exporterTest() { Assimp::Importer importer; Assimp::Exporter exporter; @@ -64,12 +65,16 @@ public: return true; } +#endif // ASSIMP_BUILD_NO_EXPORT + }; TEST_F( utglTF2ImportExport, importglTF2FromFileTest ) { EXPECT_TRUE( importerTest() ); } +#ifndef ASSIMP_BUILD_NO_EXPORT TEST_F( utglTF2ImportExport, exportglTF2FromFileTest ) { EXPECT_TRUE( exporterTest() ); } +#endif // ASSIMP_BUILD_NO_EXPORT From 4b01ecaf109ae1bbd3ae918e30abdc67d7448a5a Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 11 Sep 2017 16:31:40 -0400 Subject: [PATCH 138/305] Remove simple gltf2 export unit test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Segfaults on Linux for some reason. No other tests test exporting, so it’s fine --- test/unit/utglTF2ImportExport.cpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/test/unit/utglTF2ImportExport.cpp b/test/unit/utglTF2ImportExport.cpp index fa69d648a..8e0fcbbcf 100644 --- a/test/unit/utglTF2ImportExport.cpp +++ b/test/unit/utglTF2ImportExport.cpp @@ -43,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "AbstractImportExportBase.h" #include -#include using namespace Assimp; @@ -54,27 +53,8 @@ public: const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured.gltf", 0); return nullptr != scene; } - -#ifndef ASSIMP_BUILD_NO_EXPORT - virtual bool exporterTest() { - Assimp::Importer importer; - Assimp::Exporter exporter; - const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured.gltf", 0 ); - EXPECT_NE( nullptr, scene ); - EXPECT_EQ( aiReturn_SUCCESS, exporter.Export( scene, "gltf2", ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured_out.gltf" ) ); - - return true; - } -#endif // ASSIMP_BUILD_NO_EXPORT - }; TEST_F( utglTF2ImportExport, importglTF2FromFileTest ) { EXPECT_TRUE( importerTest() ); } - -#ifndef ASSIMP_BUILD_NO_EXPORT -TEST_F( utglTF2ImportExport, exportglTF2FromFileTest ) { - EXPECT_TRUE( exporterTest() ); -} -#endif // ASSIMP_BUILD_NO_EXPORT From 023cb27784e61ab92f915df49f18675c28a40df2 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 12 Sep 2017 09:57:58 -0400 Subject: [PATCH 139/305] Revert "Remove simple gltf2 export unit test" This reverts commit 4b01ecaf109ae1bbd3ae918e30abdc67d7448a5a. --- test/unit/utglTF2ImportExport.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/utglTF2ImportExport.cpp b/test/unit/utglTF2ImportExport.cpp index 8e0fcbbcf..fa69d648a 100644 --- a/test/unit/utglTF2ImportExport.cpp +++ b/test/unit/utglTF2ImportExport.cpp @@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "AbstractImportExportBase.h" #include +#include using namespace Assimp; @@ -53,8 +54,27 @@ public: const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured.gltf", 0); return nullptr != scene; } + +#ifndef ASSIMP_BUILD_NO_EXPORT + virtual bool exporterTest() { + Assimp::Importer importer; + Assimp::Exporter exporter; + const aiScene *scene = importer.ReadFile( ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured.gltf", 0 ); + EXPECT_NE( nullptr, scene ); + EXPECT_EQ( aiReturn_SUCCESS, exporter.Export( scene, "gltf2", ASSIMP_TEST_MODELS_DIR "/glTF2/BoxTextured-glTF/BoxTextured_out.gltf" ) ); + + return true; + } +#endif // ASSIMP_BUILD_NO_EXPORT + }; TEST_F( utglTF2ImportExport, importglTF2FromFileTest ) { EXPECT_TRUE( importerTest() ); } + +#ifndef ASSIMP_BUILD_NO_EXPORT +TEST_F( utglTF2ImportExport, exportglTF2FromFileTest ) { + EXPECT_TRUE( exporterTest() ); +} +#endif // ASSIMP_BUILD_NO_EXPORT From b0da0796c8191bc42e6cfecb765a9a4e11595a42 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Tue, 12 Sep 2017 10:07:15 -0400 Subject: [PATCH 140/305] Fix Segfault caused by losing pointer to std::string Keep std::string alive --- code/glTF2Asset.h | 14 +++++++------- code/glTF2Exporter.cpp | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index b093ec027..8604cd2a1 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -164,7 +164,7 @@ namespace glTF2 //! Magic number for GLB files #define AI_GLB_MAGIC_NUMBER "glTF" - #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0,0 + #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0, 0 #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR "$mat.gltf.pbrMetallicRoughness.roughnessFactor", 0, 0 #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLICROUGHNESS_TEXTURE aiTextureType_UNKNOWN, 0 #define AI_MATKEY_GLTF_ALPHAMODE "$mat.gltf.alphaMode", 0, 0 @@ -172,15 +172,15 @@ namespace glTF2 #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS "$mat.gltf.pbrSpecularGlossiness", 0, 0 #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_FACTOR "$clr.diffuse", 0, 1 #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULAR_FACTOR "$clr.specular", 0, 1 - #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_GLOSSINESS_FACTOR "$mat.gltf.pbrMetallicRoughness.glossinessFactor", 0,0 + #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_GLOSSINESS_FACTOR "$mat.gltf.pbrMetallicRoughness.glossinessFactor", 0, 0 #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_DIFFUSE_TEXTURE aiTextureType_DIFFUSE, 1 #define AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS_SPECULARGLOSSINESS_TEXTURE aiTextureType_UNKNOWN, 1 - #define _AI_MATKEY_GLTF_TEXTURE_TEXCOORD_BASE (std::string(_AI_MATKEY_TEXTURE_BASE) + ".texCoord").c_str() - #define _AI_MATKEY_GLTF_MAPPINGNAME_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "name").c_str() - #define _AI_MATKEY_GLTF_MAPPINGID_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "id").c_str() - #define _AI_MATKEY_GLTF_MAPPINGFILTER_MAG_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermag").c_str() - #define _AI_MATKEY_GLTF_MAPPINGFILTER_MIN_BASE (std::string(_AI_MATKEY_MAPPING_BASE) + "filtermin").c_str() + #define _AI_MATKEY_GLTF_TEXTURE_TEXCOORD_BASE "$tex.file.texCoord" + #define _AI_MATKEY_GLTF_MAPPINGNAME_BASE "$tex.mappingname" + #define _AI_MATKEY_GLTF_MAPPINGID_BASE "$tex.mappingid" + #define _AI_MATKEY_GLTF_MAPPINGFILTER_MAG_BASE "$tex.mappingfiltermag" + #define _AI_MATKEY_GLTF_MAPPINGFILTER_MIN_BASE "$tex.mappingfiltermin" #define AI_MATKEY_GLTF_TEXTURE_TEXCOORD _AI_MATKEY_GLTF_TEXTURE_TEXCOORD_BASE, type, N #define AI_MATKEY_GLTF_MAPPINGNAME(type, N) _AI_MATKEY_GLTF_MAPPINGNAME_BASE, type, N diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index e2b46b1f7..9a0e0cca1 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -276,16 +276,16 @@ void glTF2Exporter::GetTexSampler(const aiMaterial* mat, Ref texture, a void glTF2Exporter::GetMatTexProp(const aiMaterial* mat, unsigned int& prop, const char* propName, aiTextureType tt, unsigned int slot) { - const char* key = (std::string(_AI_MATKEY_TEXTURE_BASE) + "." + propName).c_str(); + std::string textureKey = std::string(_AI_MATKEY_TEXTURE_BASE) + "." + propName; - mat->Get(key, tt, slot, prop); + mat->Get(textureKey.c_str(), tt, slot, prop); } void glTF2Exporter::GetMatTexProp(const aiMaterial* mat, float& prop, const char* propName, aiTextureType tt, unsigned int slot) { - const char* key = (std::string(_AI_MATKEY_TEXTURE_BASE) + "." + propName).c_str(); + std::string textureKey = std::string(_AI_MATKEY_TEXTURE_BASE) + "." + propName; - mat->Get(key, tt, slot, prop); + mat->Get(textureKey.c_str(), tt, slot, prop); } void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt, unsigned int slot = 0) From eca008d5ece5a8087a158b5f8d1651938e8fac7b Mon Sep 17 00:00:00 2001 From: John Senneker Date: Tue, 12 Sep 2017 11:55:22 -0400 Subject: [PATCH 141/305] Properly move string passed to JSON writer --- code/glTF2AssetWriter.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index c91313ab4..7c0b435f2 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -300,7 +300,7 @@ namespace glTF2 { } if (m.alphaMode != "OPAQUE") { - obj.AddMember("alphaMode", m.alphaMode, w.mAl); + obj.AddMember("alphaMode", Value(m.alphaMode, w.mAl).Move(), w.mAl); } if (m.doubleSided) { From 933bbb4f1c4752ee7bed77029a8604aa8f3e7bc6 Mon Sep 17 00:00:00 2001 From: John Senneker Date: Tue, 12 Sep 2017 11:55:52 -0400 Subject: [PATCH 142/305] Manually read alphaMode material property --- code/glTF2Exporter.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 9a0e0cca1..a3940e9ea 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -424,7 +424,20 @@ void glTF2Exporter::ExportMaterials() mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); mat->Get(AI_MATKEY_GLTF_ALPHACUTOFF, m->alphaCutoff); - if (mat->Get(AI_MATKEY_GLTF_ALPHAMODE, m->alphaMode) != AI_SUCCESS) { + bool foundAlphaMode = false; + for (size_t i = 0; i < mat->mNumProperties; ++i) { + aiMaterialProperty *prop = mat->mProperties[i]; + if (prop->mKey != aiString("$mat.gltf.alphaMode")) + continue; + + std::string alphaMode; + for (size_t c = 0; c < prop->mDataLength; ++c) + alphaMode += prop->mData[c]; + m->alphaMode = alphaMode; + foundAlphaMode = true; + } + + if (!foundAlphaMode) { float opacity; if (mat->Get(AI_MATKEY_OPACITY, opacity) == AI_SUCCESS) { @@ -435,7 +448,7 @@ void glTF2Exporter::ExportMaterials() } } - bool hasPbrSpecularGlossiness; + bool hasPbrSpecularGlossiness = false; mat->Get(AI_MATKEY_GLTF_PBRSPECULARGLOSSINESS, hasPbrSpecularGlossiness); if (hasPbrSpecularGlossiness) { From 190f034e38bc130461df3f26b0f8fe1b650658aa Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 12 Sep 2017 18:57:44 +0300 Subject: [PATCH 143/305] Add AddressSanitizer option to CMake --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 690351e8b..260b78698 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,10 @@ OPTION ( ASSIMP_COVERALLS "Enable this to measure test coverage." OFF ) +OPTION ( ASSIMP_ASAN + "Enable AddressSanitizer." + OFF +) OPTION ( SYSTEM_IRRXML "Use system installed Irrlicht/IrrXML library." OFF @@ -212,6 +216,11 @@ if (ASSIMP_COVERALLS) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") endif() +if (ASSIMP_ASAN) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") +endif() + INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) From cbedc448c6b793d664f9357c5d4559a6da1a7c23 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 12 Sep 2017 18:59:38 +0200 Subject: [PATCH 144/305] closes https://github.com/assimp/assimp/issues/1426: add Defines.h to include folder for install. --- code/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 236e6d7ef..b9f07ce86 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -61,6 +61,7 @@ SET( PUBLIC_HEADERS ${HEADER_PATH}/color4.inl ${CMAKE_CURRENT_BINARY_DIR}/../include/assimp/config.h ${HEADER_PATH}/defs.h + ${HEADER_PATH}/Defines.h ${HEADER_PATH}/cfileio.h ${HEADER_PATH}/light.h ${HEADER_PATH}/material.h From e3163ec15eaf3df28bbe0956e9ba2687f7e42396 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 12 Sep 2017 19:07:17 +0200 Subject: [PATCH 145/305] FBX: fix some minor findings. --- code/FBXAnimation.cpp | 25 ++++++---------- code/FBXCompileConfig.h | 2 +- code/FBXConverter.cpp | 17 ++++------- code/FBXDocument.cpp | 60 ++++++++++++--------------------------- code/FBXDocument.h | 33 ++++----------------- code/FBXModel.cpp | 3 -- code/FBXNodeAttribute.cpp | 15 ++++------ code/FBXParser.cpp | 10 ++----- code/FBXParser.h | 19 ------------- 9 files changed, 47 insertions(+), 137 deletions(-) diff --git a/code/FBXAnimation.cpp b/code/FBXAnimation.cpp index e905b5bfc..6f9613f0d 100644 --- a/code/FBXAnimation.cpp +++ b/code/FBXAnimation.cpp @@ -87,17 +87,16 @@ AnimationCurve::AnimationCurve(uint64_t id, const Element& element, const std::s } } - // ------------------------------------------------------------------------------------------------ AnimationCurve::~AnimationCurve() { - + // empty } - // ------------------------------------------------------------------------------------------------ -AnimationCurveNode::AnimationCurveNode(uint64_t id, const Element& element, const std::string& name, const Document& doc, - const char* const * target_prop_whitelist /*= NULL*/, size_t whitelist_size /*= 0*/) +AnimationCurveNode::AnimationCurveNode(uint64_t id, const Element& element, const std::string& name, + const Document& doc, const char* const * target_prop_whitelist /*= NULL*/, + size_t whitelist_size /*= 0*/) : Object(id, element, name) , target() , doc(doc) @@ -154,18 +153,16 @@ AnimationCurveNode::AnimationCurveNode(uint64_t id, const Element& element, cons props = GetPropertyTable(doc,"AnimationCurveNode.FbxAnimCurveNode",element,sc,false); } - // ------------------------------------------------------------------------------------------------ AnimationCurveNode::~AnimationCurveNode() { - + // empty } - // ------------------------------------------------------------------------------------------------ const AnimationCurveMap& AnimationCurveNode::Curves() const { - if(curves.empty()) { + if ( curves.empty() ) { // resolve attached animation curves const std::vector& conns = doc.GetConnectionsByDestinationSequenced(ID(),"AnimationCurve"); @@ -195,7 +192,6 @@ const AnimationCurveMap& AnimationCurveNode::Curves() const return curves; } - // ------------------------------------------------------------------------------------------------ AnimationLayer::AnimationLayer(uint64_t id, const Element& element, const std::string& name, const Document& doc) : Object(id, element, name) @@ -207,14 +203,12 @@ AnimationLayer::AnimationLayer(uint64_t id, const Element& element, const std::s props = GetPropertyTable(doc,"AnimationLayer.FbxAnimLayer",element,sc, true); } - // ------------------------------------------------------------------------------------------------ AnimationLayer::~AnimationLayer() { - + // empty } - // ------------------------------------------------------------------------------------------------ AnimationCurveNodeList AnimationLayer::Nodes(const char* const * target_prop_whitelist /*= NULL*/, size_t whitelist_size /*= 0*/) const @@ -298,14 +292,13 @@ AnimationStack::AnimationStack(uint64_t id, const Element& element, const std::s } } - // ------------------------------------------------------------------------------------------------ AnimationStack::~AnimationStack() { - + // empty } } //!FBX } //!Assimp -#endif +#endif // ASSIMP_BUILD_NO_FBX_IMPORTER diff --git a/code/FBXCompileConfig.h b/code/FBXCompileConfig.h index d4d40b056..56aa1c787 100644 --- a/code/FBXCompileConfig.h +++ b/code/FBXCompileConfig.h @@ -66,4 +66,4 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # endif #endif -#endif +#endif // INCLUDED_AI_FBX_COMPILECONFIG_H diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index ebe7ac4e6..ff355100f 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -55,9 +55,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "StringComparison.h" #include + #include #include - #include #include @@ -931,7 +931,7 @@ bool Converter::NeedsComplexTransformationChain( const Model& model ) const TransformationComp comp = static_cast< TransformationComp >( i ); if ( comp == TransformationComp_Rotation || comp == TransformationComp_Scaling || comp == TransformationComp_Translation || - comp == TransformationComp_GeometricScaling || comp == TransformationComp_GeometricRotation || comp == TransformationComp_GeometricTranslation ) { + comp == TransformationComp_GeometricScaling || comp == TransformationComp_GeometricRotation || comp == TransformationComp_GeometricTranslation ) { continue; } @@ -949,8 +949,7 @@ std::string Converter::NameTransformationChainNode( const std::string& name, Tra return name + std::string( MAGIC_NODE_TAG ) + "_" + NameTransformationComp( comp ); } -void Converter::GenerateTransformationNodeChain( const Model& model, - std::vector& output_nodes ) +void Converter::GenerateTransformationNodeChain( const Model& model, std::vector& output_nodes ) { const PropertyTable& props = model.Props(); const Model::RotOrder rot = model.RotationOrder(); @@ -3119,7 +3118,6 @@ void Converter::InterpolateKeys( aiVectorKey* valOut, const KeyTimeList& keys, c } } - void Converter::InterpolateKeys( aiQuatKey* valOut, const KeyTimeList& keys, const KeyFrameListList& inputs, const aiVector3D& def_value, double& maxTime, @@ -3140,7 +3138,6 @@ void Converter::InterpolateKeys( aiQuatKey* valOut, const KeyTimeList& keys, con valOut[ i ].mTime = temp[ i ].mTime; - GetRotationMatrix( order, temp[ i ].mValue, m ); aiQuaternion quat = aiQuaternion( aiMatrix3x3( m ) ); @@ -3159,7 +3156,6 @@ void Converter::InterpolateKeys( aiQuatKey* valOut, const KeyTimeList& keys, con } } - void Converter::ConvertTransformOrder_TRStoSRT( aiQuatKey* out_quat, aiVectorKey* out_scale, aiVectorKey* out_translation, const KeyFrameListList& scaling, @@ -3218,7 +3214,6 @@ void Converter::ConvertTransformOrder_TRStoSRT( aiQuatKey* out_quat, aiVectorKey } } - aiQuaternion Converter::EulerToQuaternion( const aiVector3D& rot, Model::RotOrder order ) { aiMatrix4x4 m; @@ -3227,7 +3222,6 @@ aiQuaternion Converter::EulerToQuaternion( const aiVector3D& rot, Model::RotOrde return aiQuaternion( aiMatrix3x3( m ) ); } - void Converter::ConvertScaleKeys( aiNodeAnim* na, const std::vector& nodes, const LayerMap& /*layers*/, int64_t start, int64_t stop, double& maxTime, @@ -3248,7 +3242,6 @@ void Converter::ConvertScaleKeys( aiNodeAnim* na, const std::vectormScalingKeys, keys, inputs, aiVector3D( 1.0f, 1.0f, 1.0f ), maxTime, minTime ); } - void Converter::ConvertTranslationKeys( aiNodeAnim* na, const std::vector& nodes, const LayerMap& /*layers*/, int64_t start, int64_t stop, @@ -3267,7 +3260,6 @@ void Converter::ConvertTranslationKeys( aiNodeAnim* na, const std::vectormPositionKeys, keys, inputs, aiVector3D( 0.0f, 0.0f, 0.0f ), maxTime, minTime ); } - void Converter::ConvertRotationKeys( aiNodeAnim* na, const std::vector& nodes, const LayerMap& /*layers*/, int64_t start, int64_t stop, @@ -3289,7 +3281,8 @@ void Converter::ConvertRotationKeys( aiNodeAnim* na, const std::vectormMeshes && !out->mNumMeshes ); + ai_assert( !out->mMeshes ); + ai_assert( !out->mNumMeshes ); // note: the trailing () ensures initialization with NULL - not // many C++ users seem to know this, so pointing it out to avoid diff --git a/code/FBXDocument.cpp b/code/FBXDocument.cpp index b19ce4cd0..9592bf31f 100644 --- a/code/FBXDocument.cpp +++ b/code/FBXDocument.cpp @@ -70,13 +70,13 @@ LazyObject::LazyObject(uint64_t id, const Element& element, const Document& doc) , id(id) , flags() { - + // empty } // ------------------------------------------------------------------------------------------------ LazyObject::~LazyObject() { - + // empty } // ------------------------------------------------------------------------------------------------ @@ -232,16 +232,15 @@ Object::Object(uint64_t id, const Element& element, const std::string& name) , name(name) , id(id) { - + // empty } // ------------------------------------------------------------------------------------------------ Object::~Object() { - + // empty } - // ------------------------------------------------------------------------------------------------ FileGlobalSettings::FileGlobalSettings(const Document& doc, std::shared_ptr props) : props(props) @@ -361,7 +360,6 @@ void Document::ReadGlobalSettings() globals.reset(new FileGlobalSettings(*this, props)); } - // ------------------------------------------------------------------------------------------------ void Document::ReadObjects() { @@ -387,7 +385,6 @@ void Document::ReadObjects() } const char* err; - const uint64_t id = ParseTokenAsID(*tok[0], err); if(err) { DOMError(err,el.second); @@ -469,8 +466,6 @@ void Document::ReadPropertyTemplates() } } - - // ------------------------------------------------------------------------------------------------ void Document::ReadConnections() { @@ -482,7 +477,6 @@ void Document::ReadConnections() } uint64_t insertionOrder = 0l; - const Scope& sconns = *econns->Compound(); const ElementCollection conns = sconns.GetCollection("C"); for(ElementMap::const_iterator it = conns.first; it != conns.second; ++it) { @@ -491,7 +485,9 @@ void Document::ReadConnections() // PP = property-property connection, ignored for now // (tokens: "PP", ID1, "Property1", ID2, "Property2") - if(type == "PP") continue; + if ( type == "PP" ) { + continue; + } const uint64_t src = ParseTokenAsID(GetRequiredToken(el,1)); const uint64_t dest = ParseTokenAsID(GetRequiredToken(el,2)); @@ -518,11 +514,10 @@ void Document::ReadConnections() } } - // ------------------------------------------------------------------------------------------------ const std::vector& Document::AnimationStacks() const { - if (!animationStacksResolved.empty() || !animationStacks.size()) { + if (!animationStacksResolved.empty() || animationStacks.empty()) { return animationStacksResolved; } @@ -540,7 +535,6 @@ const std::vector& Document::AnimationStacks() const return animationStacksResolved; } - // ------------------------------------------------------------------------------------------------ LazyObject* Document::GetObject(uint64_t id) const { @@ -551,8 +545,7 @@ LazyObject* Document::GetObject(uint64_t id) const #define MAX_CLASSNAMES 6 // ------------------------------------------------------------------------------------------------ -std::vector Document::GetConnectionsSequenced(uint64_t id, - const ConnectionMap& conns) const +std::vector Document::GetConnectionsSequenced(uint64_t id, const ConnectionMap& conns) const { std::vector temp; @@ -569,7 +562,6 @@ std::vector Document::GetConnectionsSequenced(uint64_t id, return temp; // NRVO should handle this } - // ------------------------------------------------------------------------------------------------ std::vector Document::GetConnectionsSequenced(uint64_t id, bool is_src, const ConnectionMap& conns, @@ -578,17 +570,17 @@ std::vector Document::GetConnectionsSequenced(uint64_t id, bo { ai_assert(classnames); - ai_assert(count != 0 && count <= MAX_CLASSNAMES); + ai_assert( count != 0 ); + ai_assert( count <= MAX_CLASSNAMES); size_t lenghts[MAX_CLASSNAMES]; const size_t c = count; for (size_t i = 0; i < c; ++i) { - lenghts[i] = strlen(classnames[i]); + lenghts[ i ] = strlen(classnames[i]); } std::vector temp; - const std::pair range = conns.equal_range(id); @@ -620,49 +612,40 @@ std::vector Document::GetConnectionsSequenced(uint64_t id, bo return temp; // NRVO should handle this } - // ------------------------------------------------------------------------------------------------ std::vector Document::GetConnectionsBySourceSequenced(uint64_t source) const { return GetConnectionsSequenced(source, ConnectionsBySource()); } - - // ------------------------------------------------------------------------------------------------ -std::vector Document::GetConnectionsBySourceSequenced(uint64_t dest, - const char* classname) const +std::vector Document::GetConnectionsBySourceSequenced(uint64_t dest, const char* classname) const { const char* arr[] = {classname}; return GetConnectionsBySourceSequenced(dest, arr,1); } - - // ------------------------------------------------------------------------------------------------ -std::vector Document::GetConnectionsBySourceSequenced(uint64_t source, - const char* const* classnames, size_t count) const +std::vector Document::GetConnectionsBySourceSequenced(uint64_t source, + const char* const* classnames, size_t count) const { return GetConnectionsSequenced(source, true, ConnectionsBySource(),classnames, count); } - // ------------------------------------------------------------------------------------------------ std::vector Document::GetConnectionsByDestinationSequenced(uint64_t dest, - const char* classname) const + const char* classname) const { const char* arr[] = {classname}; return GetConnectionsByDestinationSequenced(dest, arr,1); } - // ------------------------------------------------------------------------------------------------ std::vector Document::GetConnectionsByDestinationSequenced(uint64_t dest) const { return GetConnectionsSequenced(dest, ConnectionsByDestination()); } - // ------------------------------------------------------------------------------------------------ std::vector Document::GetConnectionsByDestinationSequenced(uint64_t dest, const char* const* classnames, size_t count) const @@ -671,10 +654,9 @@ std::vector Document::GetConnectionsByDestinationSequenced(ui return GetConnectionsSequenced(dest, false, ConnectionsByDestination(),classnames, count); } - // ------------------------------------------------------------------------------------------------ Connection::Connection(uint64_t insertionOrder, uint64_t src, uint64_t dest, const std::string& prop, - const Document& doc) + const Document& doc) : insertionOrder(insertionOrder) , prop(prop) @@ -687,14 +669,12 @@ Connection::Connection(uint64_t insertionOrder, uint64_t src, uint64_t dest, co ai_assert(!dest || doc.Objects().find(dest) != doc.Objects().end()); } - // ------------------------------------------------------------------------------------------------ Connection::~Connection() { - + // empty } - // ------------------------------------------------------------------------------------------------ LazyObject& Connection::LazySourceObject() const { @@ -703,7 +683,6 @@ LazyObject& Connection::LazySourceObject() const return *lazy; } - // ------------------------------------------------------------------------------------------------ LazyObject& Connection::LazyDestinationObject() const { @@ -712,7 +691,6 @@ LazyObject& Connection::LazyDestinationObject() const return *lazy; } - // ------------------------------------------------------------------------------------------------ const Object* Connection::SourceObject() const { @@ -721,7 +699,6 @@ const Object* Connection::SourceObject() const return lazy->Get(); } - // ------------------------------------------------------------------------------------------------ const Object* Connection::DestinationObject() const { @@ -734,4 +711,3 @@ const Object* Connection::DestinationObject() const } // !Assimp #endif - diff --git a/code/FBXDocument.h b/code/FBXDocument.h index 5c8bc610f..af89f53d5 100644 --- a/code/FBXDocument.h +++ b/code/FBXDocument.h @@ -338,12 +338,7 @@ public: class Model : public Object { public: - Model(uint64_t id, const Element& element, const Document& doc, const std::string& name); - virtual ~Model(); - -public: - enum RotOrder - { + enum RotOrder { RotOrder_EulerXYZ = 0, RotOrder_EulerXZY, RotOrder_EulerYZX, @@ -357,8 +352,7 @@ public: }; - enum TransformInheritance - { + enum TransformInheritance { TransformInheritance_RrSs = 0, TransformInheritance_RSrs, TransformInheritance_Rrs, @@ -366,7 +360,10 @@ public: TransformInheritance_MAX // end-of-enum sentinel }; -public: + Model(uint64_t id, const Element& element, const Document& doc, const std::string& name); + + virtual ~Model(); + fbx_simple_property(QuaternionInterpolate, int, 0) fbx_simple_property(RotationOffset, aiVector3D, aiVector3D()) @@ -443,7 +440,6 @@ public: fbx_simple_property(LODBox, bool, false) fbx_simple_property(Freeze, bool, false) -public: const std::string& Shading() const { return shading; } @@ -462,13 +458,11 @@ public: return materials; } - /** Get geometry links */ const std::vector& GetGeometry() const { return geometry; } - /** Get node attachments */ const std::vector& GetAttributes() const { return attributes; @@ -477,7 +471,6 @@ public: /** convenience method to check if the node has a Null node marker */ bool IsNull() const; - private: void ResolveLinks(const Element& element, const Document& doc); @@ -805,7 +798,6 @@ private: typedef std::vector AnimationCurveNodeList; - /** Represents a FBX animation layer (i.e. a list of node animations) */ class AnimationLayer : public Object { @@ -828,10 +820,8 @@ private: const Document& doc; }; - typedef std::vector AnimationLayerList; - /** Represents a FBX animation stack (i.e. a list of animation layers) */ class AnimationStack : public Object { @@ -839,7 +829,6 @@ public: AnimationStack(uint64_t id, const Element& element, const std::string& name, const Document& doc); virtual ~AnimationStack(); -public: fbx_simple_property(LocalStart, int64_t, 0L) fbx_simple_property(LocalStop, int64_t, 0L) fbx_simple_property(ReferenceStart, int64_t, 0L) @@ -879,7 +868,6 @@ private: typedef std::vector WeightArray; typedef std::vector WeightIndexArray; - /** DOM class for skin deformer clusters (aka subdeformers) */ class Cluster : public Deformer { @@ -924,8 +912,6 @@ private: const Model* node; }; - - /** DOM class for skin deformers */ class Skin : public Deformer { @@ -1009,10 +995,8 @@ public: typedef std::map ObjectMap; typedef std::fbx_unordered_map > PropertyTemplateMap; - typedef std::multimap ConnectionMap; - /** DOM class for global document settings, a single instance per document can * be accessed via Document.Globals(). */ class FileGlobalSettings @@ -1074,9 +1058,6 @@ private: const Document& doc; }; - - - /** DOM root for a FBX file */ class Document { @@ -1154,8 +1135,6 @@ private: const ConnectionMap&, const char* const* classnames, size_t count) const; - -private: void ReadHeader(); void ReadObjects(); void ReadPropertyTemplates(); diff --git a/code/FBXModel.cpp b/code/FBXModel.cpp index 90939c142..3d694d7d8 100644 --- a/code/FBXModel.cpp +++ b/code/FBXModel.cpp @@ -77,14 +77,12 @@ Model::Model(uint64_t id, const Element& element, const Document& doc, const std ResolveLinks(element,doc); } - // ------------------------------------------------------------------------------------------------ Model::~Model() { } - // ------------------------------------------------------------------------------------------------ void Model::ResolveLinks(const Element& element, const Document& doc) { @@ -132,7 +130,6 @@ void Model::ResolveLinks(const Element& element, const Document& doc) } } - // ------------------------------------------------------------------------------------------------ bool Model::IsNull() const { diff --git a/code/FBXNodeAttribute.cpp b/code/FBXNodeAttribute.cpp index bcf079c2e..545343c09 100644 --- a/code/FBXNodeAttribute.cpp +++ b/code/FBXNodeAttribute.cpp @@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { namespace FBX { - using namespace Util; +using namespace Util; // ------------------------------------------------------------------------------------------------ NodeAttribute::NodeAttribute(uint64_t id, const Element& element, const Document& doc, const std::string& name) @@ -75,7 +75,7 @@ NodeAttribute::NodeAttribute(uint64_t id, const Element& element, const Document // ------------------------------------------------------------------------------------------------ NodeAttribute::~NodeAttribute() { - + // empty } @@ -101,33 +101,30 @@ CameraSwitcher::CameraSwitcher(uint64_t id, const Element& element, const Docume } } - // ------------------------------------------------------------------------------------------------ CameraSwitcher::~CameraSwitcher() { - + // empty } - // ------------------------------------------------------------------------------------------------ Camera::Camera(uint64_t id, const Element& element, const Document& doc, const std::string& name) : NodeAttribute(id,element,doc,name) { - + // empty } - // ------------------------------------------------------------------------------------------------ Camera::~Camera() { + // empty } - // ------------------------------------------------------------------------------------------------ Light::Light(uint64_t id, const Element& element, const Document& doc, const std::string& name) : NodeAttribute(id,element,doc,name) { - + // empty } diff --git a/code/FBXParser.cpp b/code/FBXParser.cpp index 428c29a62..cf6930e22 100644 --- a/code/FBXParser.cpp +++ b/code/FBXParser.cpp @@ -224,41 +224,36 @@ Parser::Parser (const TokenList& tokens, bool is_binary) root.reset(new Scope(*this,true)); } - // ------------------------------------------------------------------------------------------------ Parser::~Parser() { + // empty } - // ------------------------------------------------------------------------------------------------ TokenPtr Parser::AdvanceToNextToken() { last = current; if (cursor == tokens.end()) { current = NULL; - } - else { + } else { current = *cursor++; } return current; } - // ------------------------------------------------------------------------------------------------ TokenPtr Parser::CurrentToken() const { return current; } - // ------------------------------------------------------------------------------------------------ TokenPtr Parser::LastToken() const { return last; } - // ------------------------------------------------------------------------------------------------ uint64_t ParseTokenAsID(const Token& t, const char*& err_out) { @@ -296,7 +291,6 @@ uint64_t ParseTokenAsID(const Token& t, const char*& err_out) return id; } - // ------------------------------------------------------------------------------------------------ size_t ParseTokenAsDim(const Token& t, const char*& err_out) { diff --git a/code/FBXParser.h b/code/FBXParser.h index b0683ccd1..4d3766d70 100644 --- a/code/FBXParser.h +++ b/code/FBXParser.h @@ -85,12 +85,9 @@ typedef std::pair Element class Element { public: - Element(const Token& key_token, Parser& parser); ~Element(); -public: - const Scope* Compound() const { return compound.get(); } @@ -104,14 +101,11 @@ public: } private: - const Token& key_token; TokenList tokens; std::unique_ptr compound; }; - - /** FBX data entity that consists of a 'scope', a collection * of not necessarily unique #Element instances. * @@ -125,14 +119,10 @@ private: * @endverbatim */ class Scope { - public: - Scope(Parser& parser, bool topLevel = false); ~Scope(); -public: - const Element* operator[] (const std::string& index) const { ElementMap::const_iterator it = elements.find(index); return it == elements.end() ? NULL : (*it).second; @@ -158,28 +148,23 @@ public: } private: - ElementMap elements; }; - /** FBX parsing class, takes a list of input tokens and generates a hierarchy * of nested #Scope instances, representing the fbx DOM.*/ class Parser { public: - /** Parse given a token list. Does not take ownership of the tokens - * the objects must persist during the entire parser lifetime */ Parser (const TokenList& tokens,bool is_binary); ~Parser(); -public: const Scope& GetRootScope() const { return *root.get(); } - bool IsBinary() const { return is_binary; } @@ -233,8 +218,6 @@ void ParseVectorDataArray(std::vector& out, const Element& el); void ParseVectorDataArray(std::vector& out, const Element& e); void ParseVectorDataArray(std::vector& out, const Element& el); - - // extract a required element from a scope, abort if the element cannot be found const Element& GetRequiredElement(const Scope& sc, const std::string& index, const Element* element = NULL); @@ -243,8 +226,6 @@ const Scope& GetRequiredScope(const Element& el); // get token at a particular index const Token& GetRequiredToken(const Element& el, unsigned int index); - - // read a 4x4 matrix from an array of 16 floats aiMatrix4x4 ReadMatrix(const Element& element); From 6a3b030094481c3a901fbc8f570bf889246c9e09 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 12 Sep 2017 21:16:59 +0200 Subject: [PATCH 146/305] MDP: fix encoding issues. --- code/MMDPmdParser.h | 74 -------------------------- code/MMDPmxParser.cpp | 3 -- code/MMDPmxParser.h | 120 ------------------------------------------ code/MMDVmdParser.h | 32 ----------- 4 files changed, 229 deletions(-) diff --git a/code/MMDPmdParser.h b/code/MMDPmdParser.h index 5439b9043..586f20a5f 100644 --- a/code/MMDPmdParser.h +++ b/code/MMDPmdParser.h @@ -49,17 +49,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace pmd { - /// ヘッダ class PmdHeader { public: - /// モデル名 std::string name; - /// モデル名(英語) std::string name_english; - /// コメント std::string comment; - /// コメント(英語) std::string comment_english; bool Read(std::ifstream* stream) @@ -83,26 +78,19 @@ namespace pmd } }; - /// 頂点 class PmdVertex { public: - /// 位置 float position[3]; - /// 法線 float normal[3]; - /// UV座標 float uv[2]; - /// 関連ボーンインデックス uint16_t bone_index[2]; - /// ボーンウェイト uint8_t bone_weight; - /// エッジ不可視 bool edge_invisible; bool Read(std::ifstream* stream) @@ -117,27 +105,17 @@ namespace pmd } }; - /// 材質 class PmdMaterial { public: - /// 減衰色 float diffuse[4]; - /// 光沢度 float power; - /// 光沢色 float specular[3]; - /// 環境色 float ambient[3]; - /// トーンインデックス uint8_t toon_index; - /// エッジ uint8_t edge_flag; - /// インデックス数 uint32_t index_count; - /// テクスチャファイル名 std::string texture_filename; - /// スフィアファイル名 std::string sphere_filename; bool Read(std::ifstream* stream) @@ -180,23 +158,15 @@ namespace pmd RotationMovement }; - /// ボーン class PmdBone { public: - /// ボーン名 std::string name; - /// ボーン名(英語) std::string name_english; - /// 親ボーン番号 uint16_t parent_bone_index; - /// 末端ボーン番号 uint16_t tail_pos_bone_index; - /// ボーン種類 BoneType bone_type; - /// IKボーン番号 uint16_t ik_parent_bone_index; - /// ボーンのヘッドの位置 float bone_head_pos[3]; void Read(std::istream *stream) @@ -219,19 +189,13 @@ namespace pmd } }; - /// IK class PmdIk { public: - /// IKボーン番号 uint16_t ik_bone_index; - /// IKターゲットボーン番号 uint16_t target_bone_index; - /// 再帰回数 uint16_t interations; - /// 角度制限 float angle_limit; - /// 影響下ボーン番号 std::vector ik_child_bone_index; void Read(std::istream *stream) @@ -303,7 +267,6 @@ namespace pmd } }; - /// ボーン枠用の枠名 class PmdBoneDispName { public: @@ -338,59 +301,36 @@ namespace pmd } }; - /// 衝突形状 enum class RigidBodyShape : uint8_t { - /// 球 Sphere = 0, - /// 直方体 Box = 1, - /// カプセル Cpusel = 2 }; - /// 剛体タイプ enum class RigidBodyType : uint8_t { - /// ボーン追従 BoneConnected = 0, - /// 物理演算 Physics = 1, - /// 物理演算(Bone位置合せ) ConnectedPhysics = 2 }; - /// 剛体 class PmdRigidBody { public: - /// 名前 std::string name; - /// 関連ボーン番号 uint16_t related_bone_index; - /// グループ番号 uint8_t group_index; - /// マスク uint16_t mask; - /// 形状 RigidBodyShape shape; - /// 大きさ float size[3]; - /// 位置 float position[3]; - /// 回転 float orientation[3]; - /// 質量 float weight; - /// 移動ダンピング float linear_damping; - /// 回転ダンピング float anglar_damping; - /// 反発係数 float restitution; - /// 摩擦係数 float friction; - /// 演算方法 RigidBodyType rigid_type; void Read(std::istream *stream) @@ -414,31 +354,19 @@ namespace pmd } }; - /// 剛体の拘束 class PmdConstraint { public: - /// 名前 std::string name; - /// 剛体Aのインデックス uint32_t rigid_body_index_a; - /// 剛体Bのインデックス uint32_t rigid_body_index_b; - /// 位置 float position[3]; - /// 回転 float orientation[3]; - /// 最小移動制限 float linear_lower_limit[3]; - /// 最大移動制限 float linear_upper_limit[3]; - /// 最小回転制限 float angular_lower_limit[3]; - /// 最大回転制限 float angular_upper_limit[3]; - /// 移動に対する復元力 float linear_stiffness[3]; - /// 回転に対する復元力 float angular_stiffness[3]; void Read(std::istream *stream) @@ -459,7 +387,6 @@ namespace pmd } }; - /// PMDモデル class PmdModel { public: @@ -491,7 +418,6 @@ namespace pmd return result; } - /// ファイルからPmdModelを生成する static std::unique_ptr LoadFromStream(std::ifstream *stream) { auto result = mmd::make_unique(); diff --git a/code/MMDPmxParser.cpp b/code/MMDPmxParser.cpp index 6a76c2c7f..c38bdf49a 100644 --- a/code/MMDPmxParser.cpp +++ b/code/MMDPmxParser.cpp @@ -45,7 +45,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace pmx { - /// インデックス値を読み込む int ReadIndex(std::istream *stream, int size) { switch (size) @@ -79,7 +78,6 @@ namespace pmx } } - /// 文字列を読み込む std::string ReadString(std::istream *stream, uint8_t encoding) { int size; @@ -607,7 +605,6 @@ namespace pmx this->joints[i].Read(stream, &setting); } - //// ソフトボディ //if (this->version == 2.1f) //{ // stream->read((char*) &this->soft_body_count, sizeof(int)); diff --git a/code/MMDPmxParser.h b/code/MMDPmxParser.h index 4aff7f8eb..989cffed6 100644 --- a/code/MMDPmxParser.h +++ b/code/MMDPmxParser.h @@ -49,7 +49,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace pmx { - /// インデックス設定 class PmxSetting { public: @@ -64,26 +63,17 @@ namespace pmx , rigidbody_index_size(0) {} - /// エンコード方式 uint8_t encoding; - /// 追加UV数 uint8_t uv; - /// 頂点インデックスサイズ uint8_t vertex_index_size; - /// テクスチャインデックスサイズ uint8_t texture_index_size; - /// マテリアルインデックスサイズ uint8_t material_index_size; - /// ボーンインデックスサイズ uint8_t bone_index_size; - /// モーフインデックスサイズ uint8_t morph_index_size; - /// 剛体インデックスサイズ uint8_t rigidbody_index_size; void Read(std::istream *stream); }; - /// 頂点スキニングタイプ enum class PmxVertexSkinningType : uint8_t { BDEF1 = 0, @@ -93,7 +83,6 @@ namespace pmx QDEF = 4, }; - /// 頂点スキニング class PmxVertexSkinning { public: @@ -200,7 +189,6 @@ namespace pmx void Read(std::istream *stresam, PmxSetting *setting); }; - /// 頂点 class PmxVertex { public: @@ -219,24 +207,16 @@ namespace pmx } } - /// 位置 float position[3]; - /// 法線 float normal[3]; - /// テクスチャ座標 float uv[2]; - /// 追加テクスチャ座標 float uva[4][4]; - /// スキニングタイプ PmxVertexSkinningType skinning_type; - /// スキニング std::unique_ptr skinning; - /// エッジ倍率 float edge; void Read(std::istream *stream, PmxSetting *setting); }; - /// マテリアル class PmxMaterial { public: @@ -261,42 +241,25 @@ namespace pmx } } - /// モデル名 std::string material_name; - /// モデル英名 std::string material_english_name; - /// 減衰色 float diffuse[4]; - /// 光沢色 float specular[3]; - /// 光沢度 float specularlity; - /// 環境色 float ambient[3]; - /// 描画フラグ uint8_t flag; - /// エッジ色 float edge_color[4]; - /// エッジサイズ float edge_size; - /// アルベドテクスチャインデックス int diffuse_texture_index; - /// スフィアテクスチャインデックス int sphere_texture_index; - /// スフィアテクスチャ演算モード uint8_t sphere_op_mode; - /// 共有トゥーンフラグ uint8_t common_toon_flag; - /// トゥーンテクスチャインデックス int toon_texture_index; - /// メモ std::string memo; - /// 頂点インデックス数 int index_count; void Read(std::istream *stream, PmxSetting *setting); }; - /// リンク class PmxIkLink { public: @@ -310,18 +273,13 @@ namespace pmx } } - /// リンクボーンインデックス int link_target; - /// 角度制限 uint8_t angle_lock; - /// 最大制限角度 float max_radian[3]; - /// 最小制限角度 float min_radian[3]; void Read(std::istream *stream, PmxSetting *settingn); }; - /// ボーン class PmxBone { public: @@ -347,43 +305,24 @@ namespace pmx } } - /// ボーン名 std::string bone_name; - /// ボーン英名 std::string bone_english_name; - /// 位置 float position[3]; - /// 親ボーンインデックス int parent_index; - /// 階層 int level; - /// ボーンフラグ uint16_t bone_flag; - /// 座標オフセット(has Target) float offset[3]; - /// 接続先ボーンインデックス(not has Target) int target_index; - /// 付与親ボーンインデックス int grant_parent_index; - /// 付与率 float grant_weight; - /// 固定軸の方向 float lock_axis_orientation[3]; - /// ローカル軸のX軸方向 float local_axis_x_orientation[3]; - /// ローカル軸のY軸方向 float local_axis_y_orientation[3]; - /// 外部親変形のkey値 int key; - /// IKターゲットボーン int ik_target_bone_index; - /// IKループ回数 int ik_loop; - /// IKループ計算時の角度制限(ラジアン) float ik_loop_angle_limit; - /// IKリンク数 int ik_link_count; - /// IKリンク std::unique_ptr ik_links; void Read(std::istream *stream, PmxSetting *setting); }; @@ -543,7 +482,6 @@ namespace pmx void Read(std::istream *stream, PmxSetting *setting); //override; }; - /// モーフ class PmxMorph { public: @@ -551,34 +489,21 @@ namespace pmx : offset_count(0) { } - /// モーフ名 std::string morph_name; - /// モーフ英名 std::string morph_english_name; - /// カテゴリ MorphCategory category; - /// モーフタイプ MorphType morph_type; - /// オフセット数 int offset_count; - /// 頂点モーフ配列 std::unique_ptr vertex_offsets; - /// UVモーフ配列 std::unique_ptr uv_offsets; - /// ボーンモーフ配列 std::unique_ptr bone_offsets; - /// マテリアルモーフ配列 std::unique_ptr material_offsets; - /// グループモーフ配列 std::unique_ptr group_offsets; - /// フリップモーフ配列 std::unique_ptr flip_offsets; - /// インパルスモーフ配列 std::unique_ptr implus_offsets; void Read(std::istream *stream, PmxSetting *setting); }; - /// 枠内要素 class PmxFrameElement { public: @@ -587,14 +512,11 @@ namespace pmx , index(0) { } - /// 要素対象 uint8_t element_target; - /// 要素対象インデックス int index; void Read(std::istream *stream, PmxSetting *setting); }; - /// 表示枠 class PmxFrame { public: @@ -603,15 +525,10 @@ namespace pmx , element_count(0) { } - /// 枠名 std::string frame_name; - /// 枠英名 std::string frame_english_name; - /// 特殊枠フラグ uint8_t frame_flag; - /// 枠内要素数 int element_count; - /// 枠内要素配列 std::unique_ptr elements; void Read(std::istream *stream, PmxSetting *setting); }; @@ -637,17 +554,11 @@ namespace pmx orientation[i] = 0.0f; } } - /// 剛体名 std::string girid_body_name; - /// 剛体英名 std::string girid_body_english_name; - /// 関連ボーンインデックス int target_bone; - /// グループ uint8_t group; - /// マスク uint16_t mask; - /// 形状 uint8_t shape; float size[3]; float position[3]; @@ -818,7 +729,6 @@ namespace pmx void Read(std::istream *stream, PmxSetting *setting); }; - /// PMXモデル class PmxModel { public: @@ -836,65 +746,35 @@ namespace pmx , soft_body_count(0) {} - /// バージョン float version; - /// 設定 PmxSetting setting; - /// モデル名 std::string model_name; - /// モデル英名 std::string model_english_name; - /// コメント std::string model_comment; - /// 英語コメント std::string model_english_comment; - /// 頂点数 int vertex_count; - /// 頂点配列 std::unique_ptr vertices; - /// インデックス数 int index_count; - /// インデックス配列 std::unique_ptr indices; - /// テクスチャ数 int texture_count; - /// テクスチャ配列 std::unique_ptr< std::string []> textures; - /// マテリアル数 int material_count; - /// マテリアル std::unique_ptr materials; - /// ボーン数 int bone_count; - /// ボーン配列 std::unique_ptr bones; - /// モーフ数 int morph_count; - /// モーフ配列 std::unique_ptr morphs; - /// 表示枠数 int frame_count; - /// 表示枠配列 std::unique_ptr frames; - /// 剛体数 int rigid_body_count; - /// 剛体配列 std::unique_ptr rigid_bodies; - /// ジョイント数 int joint_count; - /// ジョイント配列 std::unique_ptr joints; - /// ソフトボディ数 int soft_body_count; - /// ソフトボディ配列 std::unique_ptr soft_bodies; - /// モデル初期化 void Init(); - /// モデル読み込み void Read(std::istream *stream); - ///// ファイルからモデルの読み込み //static std::unique_ptr ReadFromFile(const char *filename); - ///// 入力ストリームからモデルの読み込み //static std::unique_ptr ReadFromStream(std::istream *stream); }; } diff --git a/code/MMDVmdParser.h b/code/MMDVmdParser.h index c5057ef47..778ee1161 100644 --- a/code/MMDVmdParser.h +++ b/code/MMDVmdParser.h @@ -50,19 +50,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace vmd { - /// ボーンフレーム class VmdBoneFrame { public: - /// ボーン名 std::string name; - /// フレーム番号 int frame; - /// 位置 float position[3]; - /// 回転 float orientation[4]; - /// 補間曲線 char interpolation[4][4][4]; void Read(std::istream* stream) @@ -86,15 +80,11 @@ namespace vmd } }; - /// 表情フレーム class VmdFaceFrame { public: - /// 表情名 std::string face_name; - /// 表情の重み float weight; - /// フレーム番号 uint32_t frame; void Read(std::istream* stream) @@ -114,23 +104,15 @@ namespace vmd } }; - /// カメラフレーム class VmdCameraFrame { public: - /// フレーム番号 int frame; - /// 距離 float distance; - /// 位置 float position[3]; - /// 回転 float orientation[3]; - /// 補間曲線 char interpolation[6][4]; - /// 視野角 float angle; - /// 不明データ char unknown[3]; void Read(std::istream *stream) @@ -156,15 +138,11 @@ namespace vmd } }; - /// ライトフレーム class VmdLightFrame { public: - /// フレーム番号 int frame; - /// 色 float color[3]; - /// 位置 float position[3]; void Read(std::istream* stream) @@ -182,7 +160,6 @@ namespace vmd } }; - /// IKの有効無効 class VmdIkEnable { public: @@ -190,7 +167,6 @@ namespace vmd bool enable; }; - /// IKフレーム class VmdIkFrame { public: @@ -229,23 +205,15 @@ namespace vmd } }; - /// VMDモーション class VmdMotion { public: - /// モデル名 std::string model_name; - /// バージョン int version; - /// ボーンフレーム std::vector bone_frames; - /// 表情フレーム std::vector face_frames; - /// カメラフレーム std::vector camera_frames; - /// ライトフレーム std::vector light_frames; - /// IKフレーム std::vector ik_frames; static std::unique_ptr LoadFromFile(char const *filename) From 2938a259b8c55e5f37ecacf0e2e0caed9c2d7010 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 12 Sep 2017 19:09:22 +0300 Subject: [PATCH 147/305] Enable AddressSanitizer for Linux clang build --- .travis.sh | 2 +- .travis.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.sh b/.travis.sh index 7161fd28d..64aafe922 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ASAN=$ASAN } if [ $ANDROID ]; then diff --git a/.travis.yml b/.travis.yml index 38849dae4..bb1820cc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,13 +28,13 @@ env: - secure: "lZ7pHQvl5dpZWzBQAaIMf0wqrvtcZ4wiZKeIZjf83TEsflW8+z0uTpIuN30ZV6Glth/Sq1OhLnTP5+N57fZU/1ebA5twHdvP4bS5CIUUg71/CXQZNl36xeaqvxsG/xRrdpKOsPdjAOsQ9KPTQulsX43XDLS7CasMiLvYOpqKcPc=" - PV=r8e PLATF=linux-x86_64 NDK_HOME=${TRAVIS_BUILD_DIR}/android-ndk-${PV} PATH=${PATH}:${NDK_HOME} matrix: - - os: linux LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON + - os: linux LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON ASAN=OFF compiler: gcc - - os: linux LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + - os: linux LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON compiler: clang - - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF compiler: gcc - - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF compiler: clang - os: osx osx_image: xcode8.2 From b6f122ff2c20bbd7ada5f45bdcfb4c29b5370c42 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 12 Sep 2017 19:00:44 +0300 Subject: [PATCH 148/305] Fix delete / delete[] mismatch in glTFAsset --- code/glTFAsset.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index 91d36c59b..0a0de2ce3 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -332,7 +332,7 @@ inline bool Buffer::LoadFromStream(IOStream& stream, size_t length, size_t baseO stream.Seek(baseOffset, aiOrigin_SET); } - mData.reset(new uint8_t[byteLength]); + mData.reset(new uint8_t[byteLength], std::default_delete()); if (stream.Read(mData.get(), byteLength, 1) != 1) { return false; From 2cc0a378ed156dbefec62dffdf994aa6901dc177 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Wed, 13 Sep 2017 11:23:12 -0400 Subject: [PATCH 149/305] Update glTF in list of importers and exporters --- Readme.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Readme.md b/Readme.md index a5c31881a..1eaa6b8e1 100644 --- a/Readme.md +++ b/Readme.md @@ -52,7 +52,8 @@ __Importers__: - DXF - ENFF - FBX -- GLB/GLTF +- glTF 1.0 + GLB +- glTF 2.0 - HMB - IFC-STEP - IRR / IRRMESH @@ -106,8 +107,8 @@ __Exporters__: - JSON (for WebGl, via https://github.com/acgessler/assimp2json) - ASSBIN - STEP -- glTF (partial) -- glTF2.0 +- glTF 1.0 (partial) +- glTF 2.0 (partial) ### Building ### Take a look into the `INSTALL` file. Our build system is CMake, if you used CMake before there is a good chance you know what to do. From afd6c4d57ddfa8428c42afb63a953f76621c3718 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Wed, 13 Sep 2017 21:40:44 +0200 Subject: [PATCH 150/305] Revert "Asan" --- .travis.sh | 2 +- .travis.yml | 8 ++++---- CMakeLists.txt | 9 --------- code/glTFAsset.inl | 2 +- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.travis.sh b/.travis.sh index 64aafe922..7161fd28d 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ASAN=$ASAN + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS } if [ $ANDROID ]; then diff --git a/.travis.yml b/.travis.yml index bb1820cc4..38849dae4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,13 +28,13 @@ env: - secure: "lZ7pHQvl5dpZWzBQAaIMf0wqrvtcZ4wiZKeIZjf83TEsflW8+z0uTpIuN30ZV6Glth/Sq1OhLnTP5+N57fZU/1ebA5twHdvP4bS5CIUUg71/CXQZNl36xeaqvxsG/xRrdpKOsPdjAOsQ9KPTQulsX43XDLS7CasMiLvYOpqKcPc=" - PV=r8e PLATF=linux-x86_64 NDK_HOME=${TRAVIS_BUILD_DIR}/android-ndk-${PV} PATH=${PATH}:${NDK_HOME} matrix: - - os: linux LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON ASAN=OFF + - os: linux LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON compiler: gcc - - os: linux LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + - os: linux LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF compiler: clang - - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF compiler: gcc - - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF compiler: clang - os: osx osx_image: xcode8.2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 260b78698..690351e8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,10 +78,6 @@ OPTION ( ASSIMP_COVERALLS "Enable this to measure test coverage." OFF ) -OPTION ( ASSIMP_ASAN - "Enable AddressSanitizer." - OFF -) OPTION ( SYSTEM_IRRXML "Use system installed Irrlicht/IrrXML library." OFF @@ -216,11 +212,6 @@ if (ASSIMP_COVERALLS) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") endif() -if (ASSIMP_ASAN) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") -endif() - INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index 61afebfb4..9284ccb02 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -341,7 +341,7 @@ inline bool Buffer::LoadFromStream(IOStream& stream, size_t length, size_t baseO stream.Seek(baseOffset, aiOrigin_SET); } - mData.reset(new uint8_t[byteLength], std::default_delete()); + mData.reset(new uint8_t[byteLength]); if (stream.Read(mData.get(), byteLength, 1) != 1) { return false; From 58213804ff82c5cd7deccd9478d4b0d5770c7992 Mon Sep 17 00:00:00 2001 From: vkovalev123 Date: Thu, 14 Sep 2017 14:55:02 +0300 Subject: [PATCH 151/305] Update 3DSLoader.cpp Fixed reading of CHINK_RGBF. If reading performs on x32 platform then reading will execute right, but on x64 it`s wrong because it will read 8 bytes instead 4. --- code/3DSLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/3DSLoader.cpp b/code/3DSLoader.cpp index 522bec307..704884a57 100644 --- a/code/3DSLoader.cpp +++ b/code/3DSLoader.cpp @@ -1381,7 +1381,7 @@ void Discreet3DSImporter::ParseColorChunk( aiColor3D* out, bool acceptPercent ) bGamma = true; case Discreet3DS::CHUNK_RGBF: - if (sizeof(ai_real) * 3 > diff) { + if (sizeof(float) * 3 > diff) { *out = clrError; return; } From 94a6fc78f423e18404e13ef271cc3fdbe77472ac Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Fri, 15 Sep 2017 18:27:59 +1000 Subject: [PATCH 152/305] Addressed last remaining warning under MSVC caused by use of 'deprecated' fopen. --- contrib/irrXML/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/irrXML/CMakeLists.txt b/contrib/irrXML/CMakeLists.txt index 980bd99a4..48941970a 100644 --- a/contrib/irrXML/CMakeLists.txt +++ b/contrib/irrXML/CMakeLists.txt @@ -8,6 +8,11 @@ set( IrrXML_SRCS irrXML.h ) +if ( MSVC ) + ADD_DEFINITIONS( -D_SCL_SECURE_NO_WARNINGS ) + ADD_DEFINITIONS( -D_CRT_SECURE_NO_WARNINGS ) +endif ( MSVC ) + add_library(IrrXML STATIC ${IrrXML_SRCS}) set(IRRXML_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" CACHE INTERNAL "IrrXML_Include" ) set(IRRXML_LIBRARY "IrrXML" CACHE INTERNAL "IrrXML" ) From f2e2f74d730e19ab2a705d6f96bc56816e667f16 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 10:54:02 +0300 Subject: [PATCH 153/305] Add CMake flag to treat warnings as errors --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 690351e8b..b404e6916 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,10 @@ OPTION ( ASSIMP_COVERALLS "Enable this to measure test coverage." OFF ) +OPTION ( ASSIMP_WERRRO + "Treat warnings as errors." + OFF +) OPTION ( SYSTEM_IRRXML "Use system installed Irrlicht/IrrXML library." OFF @@ -212,6 +216,11 @@ if (ASSIMP_COVERALLS) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") endif() +if (ASSIMP_WERROR) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") +endif() + INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) From 0b140db0a40fb7563964029af927785fe81cd5b2 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 11:18:02 +0300 Subject: [PATCH 154/305] glTFExporter: Silence uninitialized variable warning This is a false positive, idx_srcdata_begin is only used if comp_allow is true and in that case it's also initialized. --- code/glTFExporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index 8967a7aa7..7215a98c3 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -507,7 +507,7 @@ void glTFExporter::ExportMeshes() // Variables needed for compression. BEGIN. // Indices, not pointers - because pointer to buffer is changing while writing to it. - size_t idx_srcdata_begin;// Index of buffer before writing mesh data. Also, index of begin of coordinates array in buffer. + size_t idx_srcdata_begin = 0; // Index of buffer before writing mesh data. Also, index of begin of coordinates array in buffer. size_t idx_srcdata_normal = SIZE_MAX;// Index of begin of normals array in buffer. SIZE_MAX - mean that mesh has no normals. std::vector idx_srcdata_tc;// Array of indices. Every index point to begin of texture coordinates array in buffer. size_t idx_srcdata_ind;// Index of begin of coordinates indices array in buffer. From b9efc234d0cbc08f0d09135d4cca56298cf8adfd Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 11:22:32 +0300 Subject: [PATCH 155/305] DefaultLogger: Whitespace cleanup to fix GCC misleading indentation warning --- code/DefaultLogger.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/DefaultLogger.cpp b/code/DefaultLogger.cpp index 5a6f19544..b69e18ea9 100644 --- a/code/DefaultLogger.cpp +++ b/code/DefaultLogger.cpp @@ -253,8 +253,8 @@ void DefaultLogger::kill() // Debug message void DefaultLogger::OnDebug( const char* message ) { - if ( m_Severity == Logger::NORMAL ) - return; + if ( m_Severity == Logger::NORMAL ) + return; static const size_t Size = MAX_LOG_MESSAGE_LENGTH + 16; char msg[Size]; From f4a0ab81b1d711c1151bee390e0ed5189d2f5560 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 11:25:19 +0300 Subject: [PATCH 156/305] AssbinExporter: Add Write specialization for aiColor3D --- code/AssbinExporter.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/code/AssbinExporter.cpp b/code/AssbinExporter.cpp index 4ea261718..bd2ed0b95 100644 --- a/code/AssbinExporter.cpp +++ b/code/AssbinExporter.cpp @@ -139,6 +139,17 @@ inline size_t Write(IOStream * stream, const aiVector3D& v) return t; } +// ----------------------------------------------------------------------------------- +// Serialize a color value +template <> +inline size_t Write(IOStream * stream, const aiColor3D& v) +{ + size_t t = Write(stream,v.r); + t += Write(stream,v.g); + t += Write(stream,v.b); + return t; +} + // ----------------------------------------------------------------------------------- // Serialize a color value template <> From 046c229e486369ee52ec45db37e18a022cc511d3 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Fri, 15 Sep 2017 12:41:40 +0300 Subject: [PATCH 157/305] AssbinExporter: Fix strict aliasing violation --- code/AssbinExporter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/AssbinExporter.cpp b/code/AssbinExporter.cpp index bd2ed0b95..a6fbcd8f7 100644 --- a/code/AssbinExporter.cpp +++ b/code/AssbinExporter.cpp @@ -650,9 +650,9 @@ inline size_t WriteArray(IOStream * stream, const T* in, unsigned int size) Write(&chunk,l->mAttenuationQuadratic); } - Write(&chunk,(const aiVector3D&)l->mColorDiffuse); - Write(&chunk,(const aiVector3D&)l->mColorSpecular); - Write(&chunk,(const aiVector3D&)l->mColorAmbient); + Write(&chunk,l->mColorDiffuse); + Write(&chunk,l->mColorSpecular); + Write(&chunk,l->mColorAmbient); if (l->mType == aiLightSource_SPOT) { Write(&chunk,l->mAngleInnerCone); From f1998d52dce46b33e669f671a0672ba4dde752fc Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 12:19:03 +0300 Subject: [PATCH 158/305] Importer: Whitespace cleanup to fix GCC misleading indentation warning --- code/Importer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/Importer.cpp b/code/Importer.cpp index eb0119693..379daab02 100644 --- a/code/Importer.cpp +++ b/code/Importer.cpp @@ -831,8 +831,8 @@ const aiScene* Importer::ApplyPostProcessing(unsigned int pFlags) pimpl->mProgressHandler->UpdatePostProcess( static_cast(pimpl->mPostProcessingSteps.size()), static_cast(pimpl->mPostProcessingSteps.size()) ); // update private scene flags - if( pimpl->mScene ) - ScenePriv(pimpl->mScene)->mPPStepsApplied |= pFlags; + if( pimpl->mScene ) + ScenePriv(pimpl->mScene)->mPPStepsApplied |= pFlags; // clear any data allocated by post-process steps pimpl->mPPShared->Clean(); From 01c50394ce1e870e744ec2537c271562276f474a Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Fri, 15 Sep 2017 12:31:52 +0300 Subject: [PATCH 159/305] FBXParser: Silence uninitialized variable warnings --- code/FBXParser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/FBXParser.cpp b/code/FBXParser.cpp index cf6930e22..b5661caf8 100644 --- a/code/FBXParser.cpp +++ b/code/FBXParser.cpp @@ -281,7 +281,7 @@ uint64_t ParseTokenAsID(const Token& t, const char*& err_out) unsigned int length = static_cast(t.end() - t.begin()); ai_assert(length > 0); - const char* out; + const char* out = nullptr; const uint64_t id = strtoul10_64(t.begin(),&out,&length); if (out > t.end()) { err_out = "failed to parse ID (text)"; @@ -327,7 +327,7 @@ size_t ParseTokenAsDim(const Token& t, const char*& err_out) return 0; } - const char* out; + const char* out = nullptr; const size_t id = static_cast(strtoul10_64(t.begin() + 1,&out,&length)); if (out > t.end()) { err_out = "failed to parse ID"; @@ -440,7 +440,7 @@ int64_t ParseTokenAsInt64(const Token& t, const char*& err_out) unsigned int length = static_cast(t.end() - t.begin()); ai_assert(length > 0); - const char* out; + const char* out = nullptr; const int64_t id = strtol10_64(t.begin(), &out, &length); if (out > t.end()) { err_out = "failed to parse Int64 (text)"; From 234ffc0ad6a22c590209216f4d727252e75cb0b9 Mon Sep 17 00:00:00 2001 From: Richard Mitton Date: Fri, 15 Sep 2017 12:39:58 -0700 Subject: [PATCH 160/305] Fixed truncated material names The previous Unicode library change accidentally trimmed off the last character of SIB material names. --- code/SIBImporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/SIBImporter.cpp b/code/SIBImporter.cpp index 6cd398f48..3fa66bd77 100644 --- a/code/SIBImporter.cpp +++ b/code/SIBImporter.cpp @@ -201,7 +201,7 @@ static aiString ReadString(StreamReaderLE* stream, uint32_t numWChars) //ConvertUTF16toUTF8(&start, end, &dest, limit, lenientConversion); //*dest = '\0'; - str[str.size()-1] = '\0'; + str[str.size()] = '\0'; // Return the final string. aiString result = aiString((const char *)&str[0]); //delete[] str; From f602055da5e63675913bf3747b1b754dae31e0cd Mon Sep 17 00:00:00 2001 From: Richard Mitton Date: Fri, 15 Sep 2017 12:40:38 -0700 Subject: [PATCH 161/305] Added Silo 2.5 support Silo 2.5 bumps the version number of SIB files for no apparent reason. Doesn't appear to be any other changes to the file format. --- code/SIBImporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/SIBImporter.cpp b/code/SIBImporter.cpp index 3fa66bd77..4ae2ab7d4 100644 --- a/code/SIBImporter.cpp +++ b/code/SIBImporter.cpp @@ -827,7 +827,7 @@ static void ReadInstance(SIB* sib, StreamReaderLE* stream) static void CheckVersion(StreamReaderLE* stream) { uint32_t version = stream->GetU4(); - if ( version != 1 ) { + if ( version < 1 || version > 2 ) { throw DeadlyImportError( "SIB: Unsupported file version." ); } } From 5478d4d4c758f4f652399bb0597e5f3bc96a53c9 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 16 Sep 2017 15:22:37 +0300 Subject: [PATCH 162/305] travis: Move os declarations earlier --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38849dae4..c02be9e06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,10 @@ branches: only: - master +os: + - linux + - osx + osx_image: xcode8.3 env: @@ -50,9 +54,6 @@ script: - export COVERALLS_SERVICE_NAME=travis-ci - export COVERALLS_REPO_TOKEN=abc12345 - . ./.travis.sh -os: - - linux - - osx after_success: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then cd ${TRAVIS_BUILD_DIR} && lcov --directory . --capture --output-file coverage.info && lcov --remove coverage.info '/usr/*' 'contrib/*' 'test/*' --output-file coverage.info && lcov --list coverage.info && coveralls-lcov --source-encoding=ISO-8859-1 --repo-token=${COVERALLS_TOKEN} coverage.info ; fi From 167fb7e250a813e1f96bb7903d212a2fd490ae4b Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 16 Sep 2017 15:34:32 +0300 Subject: [PATCH 163/305] travis: Correctly minimize build matrix --- .travis.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c02be9e06..ddb3fedd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,17 +31,29 @@ env: # COVERITY_SCAN_TOKEN - secure: "lZ7pHQvl5dpZWzBQAaIMf0wqrvtcZ4wiZKeIZjf83TEsflW8+z0uTpIuN30ZV6Glth/Sq1OhLnTP5+N57fZU/1ebA5twHdvP4bS5CIUUg71/CXQZNl36xeaqvxsG/xRrdpKOsPdjAOsQ9KPTQulsX43XDLS7CasMiLvYOpqKcPc=" - PV=r8e PLATF=linux-x86_64 NDK_HOME=${TRAVIS_BUILD_DIR}/android-ndk-${PV} PATH=${PATH}:${NDK_HOME} - matrix: - - os: linux LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON + +matrix: + exclude: + - os: linux + env: + + include: + - os: linux compiler: gcc - - os: linux LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - compiler: clang - - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON + - os: linux compiler: gcc - - os: linux LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - compiler: clang + env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + - os: linux + compiler: gcc + env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + - os: linux + compiler: gcc + env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - os: osx + compiler: clang osx_image: xcode8.2 + env: install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From 798d2d063ced4a922bd41b3eb9f7630830e022e8 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 16 Sep 2017 15:52:18 +0300 Subject: [PATCH 164/305] travis: Only build with xcode 8.3 --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ddb3fedd9..6c29561e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,6 @@ os: - linux - osx -osx_image: xcode8.3 - env: global: # COVERITY_SCAN_TOKEN @@ -52,7 +50,7 @@ matrix: env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - os: osx compiler: clang - osx_image: xcode8.2 + osx_image: xcode8.3 env: install: From cf8453a21a5661e4a486c82dcc87fb5a64670d68 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 16 Sep 2017 15:53:02 +0300 Subject: [PATCH 165/305] travis: Enable ccache --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6c29561e3..a75c5e94b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ sudo: required language: cpp +cache: ccache + before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install cmake && sudo apt-get install cmake python3 && sudo apt-get install -qq freeglut3-dev libxmu-dev libxi-dev ; echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi - 'if [ "$TRAVIS_OS_NAME" = "osx" ]; then From 69f8dc9b31550594c719ebc0358b73fa76e0cc17 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 16 Sep 2017 17:59:47 +0300 Subject: [PATCH 166/305] Travis: Disable OS X build since it doesn't do anything currently --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a75c5e94b..cf0988517 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,6 @@ branches: os: - linux - - osx env: global: @@ -50,10 +49,6 @@ matrix: - os: linux compiler: gcc env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - - os: osx - compiler: clang - osx_image: xcode8.3 - env: install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From c207e745349a18da5366286d2e8a18a21e2ce41c Mon Sep 17 00:00:00 2001 From: Jeremy Cowles Date: Sun, 17 Sep 2017 10:06:57 -0700 Subject: [PATCH 167/305] Fix glTF 2.0 multi-primitive support Previously, only one primitive was supported, in fact memory was corrupted when more than one primitive was found per glTF mesh. In this change, each primitive is unrolled as a new Assimp Mesh, resulting in multiple Assimp meshes per node when multiple primitives exist per glTF mesh. This is required in the general case, since glTF primitives can have different material bindings and primitive modes. --- code/glTF2Importer.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 34f12b5af..6de482981 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -518,13 +518,22 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& } if (node.mesh) { - ainode->mNumMeshes = 1; - ainode->mMeshes = new unsigned int[1]; - int k = 0; int idx = node.mesh.GetIndex(); - for (unsigned int j = meshOffsets[idx]; j < meshOffsets[idx + 1]; ++j, ++k) { + ai_assert(idx >= 0 && idx < meshOffsets.size()); + + unsigned int offBegin = meshOffsets[idx]; + unsigned int offEnd = meshOffsets[idx + 1]; + int k = 0; + + ai_assert(offEnd >= offBegin); + + ainode->mNumMeshes = offEnd - offBegin; + ainode->mMeshes = new unsigned int[ainode->mNumMeshes]; + + for (unsigned int j = offBegin; j < offEnd; ++j, ++k) { + ai_assert(k < ainode->mNumMeshes); ainode->mMeshes[k] = j; } } From 41724ace2d0fdba04330b95d4018b499a09ea73c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 18 Sep 2017 14:10:58 +0300 Subject: [PATCH 168/305] Collada: Silence uninitialized variable warning This is a false positive. Value of 'method' is only used if 'targetMeshes' contains something and all paths through the first loop which add stuff to 'targetMeshes' also set 'method'. --- code/ColladaLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ColladaLoader.cpp b/code/ColladaLoader.cpp index 0e970b468..ccf79ed62 100644 --- a/code/ColladaLoader.cpp +++ b/code/ColladaLoader.cpp @@ -674,7 +674,7 @@ aiMesh* ColladaLoader::CreateMesh( const ColladaParser& pParser, const Collada:: // create morph target meshes if any std::vector targetMeshes; std::vector targetWeights; - Collada::MorphMethod method; + Collada::MorphMethod method = Collada::Normalized; for(std::map::const_iterator it = pParser.mControllerLibrary.begin(); it != pParser.mControllerLibrary.end(); it++) From 4652be8f18842cf3dc2cbcac1b3e8bd21115a2e4 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 18 Sep 2017 14:59:55 +0300 Subject: [PATCH 169/305] FIReader: Silence uninitialized variable warning This is a false positive. First time through the loop 'imod3' is always 0 so c1 is not used. It's also set so further iterations have a valid 'c1'. If 'value' is empty the switch doesn't look at 'c1' either since 'imod3' is still 0. --- code/FIReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/FIReader.cpp b/code/FIReader.cpp index 981d0b913..95b22a1b8 100755 --- a/code/FIReader.cpp +++ b/code/FIReader.cpp @@ -168,7 +168,7 @@ struct FIBase64ValueImpl: public FIBase64Value { if (!strValueValid) { strValueValid = true; std::ostringstream os; - uint8_t c1, c2; + uint8_t c1 = 0, c2; int imod3 = 0; std::vector::size_type valueSize = value.size(); for (std::vector::size_type i = 0; i < valueSize; ++i) { From b74fc9495aeb35346a0b941de3f984d17eebd3dc Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 18 Sep 2017 15:13:19 +0300 Subject: [PATCH 170/305] PlyLoader: Fix operator precedence issue in header check The previous version might read past end of buffer --- code/PlyLoader.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/PlyLoader.cpp b/code/PlyLoader.cpp index d348fe113..c6e862bf1 100644 --- a/code/PlyLoader.cpp +++ b/code/PlyLoader.cpp @@ -170,9 +170,10 @@ void PLYImporter::InternReadFile(const std::string& pFile, std::vector headerCheck; streamedBuffer.getNextLine(headerCheck); - if ((headerCheck.size() >= 3) && (headerCheck[0] != 'P' && headerCheck[0] != 'p') || - (headerCheck[1] != 'L' && headerCheck[1] != 'l') || - (headerCheck[2] != 'Y' && headerCheck[2] != 'y')) + if ((headerCheck.size() < 3) || + (headerCheck[0] != 'P' && headerCheck[0] != 'p') || + (headerCheck[1] != 'L' && headerCheck[1] != 'l') || + (headerCheck[2] != 'Y' && headerCheck[2] != 'y') ) { streamedBuffer.close(); throw DeadlyImportError("Invalid .ply file: Magic number \'ply\' is no there"); From 40c308af4423429452e0ee03fb3d50bc60a5d6c0 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 18 Sep 2017 15:18:45 +0300 Subject: [PATCH 171/305] glTF: Silence uninitialized variable warning This is a false positive. 'jointNamesIndex' is either set by the loop or the following conditional is false which also sets it. The undefined value is never seen by the following code. --- code/glTF2Exporter.cpp | 2 +- code/glTFExporter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index a3940e9ea..bcd1e2858 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -541,7 +541,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref nodeRef = mAsset.nodes.Get(aib->mName.C_Str()); nodeRef->jointName = nodeRef->name; - unsigned int jointNamesIndex; + unsigned int jointNamesIndex = 0; bool addJointToJointNames = true; for ( unsigned int idx_joint = 0; idx_joint < skinRef->jointNames.size(); ++idx_joint) { if (skinRef->jointNames[idx_joint]->jointName.compare(nodeRef->jointName) == 0) { diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index 7215a98c3..a5df09ac7 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -444,7 +444,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref nodeRef = mAsset.nodes.Get(aib->mName.C_Str()); nodeRef->jointName = nodeRef->id; - unsigned int jointNamesIndex; + unsigned int jointNamesIndex = 0; bool addJointToJointNames = true; for ( unsigned int idx_joint = 0; idx_joint < skinRef->jointNames.size(); ++idx_joint) { if (skinRef->jointNames[idx_joint]->jointName.compare(nodeRef->jointName) == 0) { From 982430c3ce408201ca6201b204b35e382ec0e101 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 18 Sep 2017 15:21:51 +0300 Subject: [PATCH 172/305] BlenderDNA: Silence warning about inline function which is declared but not defined It's a templated method which is meant to be specialized. The base version is never called. Just remove 'inline' to make GCC shut up. --- code/BlenderDNA.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/BlenderDNA.h b/code/BlenderDNA.h index de0a5a62f..4c7d0fc26 100644 --- a/code/BlenderDNA.h +++ b/code/BlenderDNA.h @@ -253,7 +253,7 @@ public: * a compiler complain is the result. * @param dest Destination value to be written * @param db File database, including input stream. */ - template inline void Convert (T& dest, const FileDatabase& db) const; + template void Convert (T& dest, const FileDatabase& db) const; // -------------------------------------------------------- // generic converter From 5147acfe65978fd9d5400c4d39f521c85f9a660c Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 17 Sep 2017 15:11:01 -0400 Subject: [PATCH 173/305] Revert "store node mesh vs. meshes" This reverts commit a0d97505e5a53a9aa3a734d6bd3ad790234e6e09. --- code/glTF2Asset.h | 2 +- code/glTF2Asset.inl | 9 ++++++++- code/glTF2AssetWriter.inl | 4 +--- code/glTF2Exporter.cpp | 19 ++++++++++--------- code/glTF2Importer.cpp | 28 +++++++++++++--------------- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 8604cd2a1..e2b61c646 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -781,7 +781,7 @@ namespace glTF2 struct Node : public Object { std::vector< Ref > children; - Ref mesh; + std::vector< Ref > meshes; Nullable matrix; Nullable translation; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index cbdac5d80..0a3e2b051 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -934,9 +934,16 @@ inline void Node::Read(Value& obj, Asset& r) } if (Value* mesh = FindUInt(obj, "mesh")) { + //unsigned numMeshes = (unsigned)meshes->Size(); + unsigned numMeshes = 1; + + //std::vector meshList; + + this->meshes.reserve(numMeshes); + Ref meshRef = r.meshes.Retrieve((*mesh).GetUint()); - if (meshRef) this->mesh = meshRef; + if (meshRef) this->meshes.push_back(meshRef); } if (Value* camera = FindUInt(obj, "camera")) { diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index 7c0b435f2..dd90ec576 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -417,9 +417,7 @@ namespace glTF2 { AddRefsVector(obj, "children", n.children, w.mAl); - if (n.mesh) { - obj.AddMember("mesh", n.mesh->index, w.mAl); - } + AddRefsVector(obj, "meshes", n.meshes, w.mAl); AddRefsVector(obj, "skeletons", n.skeletons, w.mAl); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index a3940e9ea..56932dba7 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -476,15 +476,16 @@ void glTF2Exporter::ExportMaterials() */ bool FindMeshNode(Ref& nodeIn, Ref& meshNode, std::string meshID) { - - if (nodeIn->mesh && meshID.compare(nodeIn->mesh->id) == 0) { - meshNode = nodeIn; - return true; + for (unsigned int i = 0; i < nodeIn->meshes.size(); ++i) { + if (meshID.compare(nodeIn->meshes[i]->id) == 0) { + meshNode = nodeIn; + return true; + } } for (unsigned int i = 0; i < nodeIn->children.size(); ++i) { if(FindMeshNode(nodeIn->children[i], meshNode, meshID)) { - return true; + return true; } } @@ -755,8 +756,8 @@ unsigned int glTF2Exporter::ExportNodeHierarchy(const aiNode* n) CopyValue(n->mTransformation, node->matrix.value); } - if (n->mNumMeshes > 0) { - node->mesh = mAsset->meshes.Get(n->mMeshes[0]); + for (unsigned int i = 0; i < n->mNumMeshes; ++i) { + node->meshes.push_back(mAsset->meshes.Get(n->mMeshes[i])); } for (unsigned int i = 0; i < n->mNumChildren; ++i) { @@ -784,8 +785,8 @@ unsigned int glTF2Exporter::ExportNode(const aiNode* n, Ref& parent) CopyValue(n->mTransformation, node->matrix.value); } - if (n->mNumMeshes > 0) { - node->mesh = mAsset->meshes.Get(n->mMeshes[0]); + for (unsigned int i = 0; i < n->mNumMeshes; ++i) { + node->meshes.push_back(mAsset->meshes.Get(n->mMeshes[i])); } for (unsigned int i = 0; i < n->mNumChildren; ++i) { diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 6de482981..d9dade3b2 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -517,24 +517,22 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& } } - if (node.mesh) { + if (!node.meshes.empty()) { + int count = 0; + for (size_t i = 0; i < node.meshes.size(); ++i) { + int idx = node.meshes[i].GetIndex(); + count += meshOffsets[idx + 1] - meshOffsets[idx]; + } + ainode->mNumMeshes = count; - int idx = node.mesh.GetIndex(); + ainode->mMeshes = new unsigned int[count]; - ai_assert(idx >= 0 && idx < meshOffsets.size()); - - unsigned int offBegin = meshOffsets[idx]; - unsigned int offEnd = meshOffsets[idx + 1]; int k = 0; - - ai_assert(offEnd >= offBegin); - - ainode->mNumMeshes = offEnd - offBegin; - ainode->mMeshes = new unsigned int[ainode->mNumMeshes]; - - for (unsigned int j = offBegin; j < offEnd; ++j, ++k) { - ai_assert(k < ainode->mNumMeshes); - ainode->mMeshes[k] = j; + for (size_t i = 0; i < node.meshes.size(); ++i) { + int idx = node.meshes[i].GetIndex(); + for (unsigned int j = meshOffsets[idx]; j < meshOffsets[idx + 1]; ++j, ++k) { + ainode->mMeshes[k] = j; + } } } From 28523232cf735a28ec794a557af48163d84174c0 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Sun, 17 Sep 2017 17:00:57 -0400 Subject: [PATCH 174/305] Merge multiple meshes in a node into one mesh with many primtives; write out only one mesh per node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To do: - clean up MergeMeshes - see if there’s a way to do this earlier in the flow --- code/glTF2AssetWriter.inl | 4 +++- code/glTF2Exporter.cpp | 22 ++++++++++++++++++++++ code/glTF2Exporter.h | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/code/glTF2AssetWriter.inl b/code/glTF2AssetWriter.inl index dd90ec576..df28cb681 100644 --- a/code/glTF2AssetWriter.inl +++ b/code/glTF2AssetWriter.inl @@ -417,7 +417,9 @@ namespace glTF2 { AddRefsVector(obj, "children", n.children, w.mAl); - AddRefsVector(obj, "meshes", n.meshes, w.mAl); + if (!n.meshes.empty()) { + obj.AddMember("mesh", n.meshes[0]->index, w.mAl); + } AddRefsVector(obj, "skeletons", n.skeletons, w.mAl); diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 56932dba7..f63c8e89c 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -110,6 +110,7 @@ glTF2Exporter::glTF2Exporter(const char* filename, IOSystem* pIOSystem, const ai } ExportMeshes(); + MergeMeshes(); ExportScene(); @@ -743,6 +744,27 @@ void glTF2Exporter::ExportMeshes() } } +void glTF2Exporter::MergeMeshes() +{ + for (unsigned int n = 0; n < mAsset->nodes.Size(); ++n) { + Ref node = mAsset->nodes.Get(n); + + unsigned int nMeshes = node->meshes.size(); + + if (nMeshes) { + Ref firstMesh = node->meshes.at(0); + + for (unsigned int m = 1; m < nMeshes; ++m) { + Ref mesh = node->meshes.at(m); + Mesh::Primitive primitive = mesh->primitives.at(0); + firstMesh->primitives.push_back(primitive); + } + + node->meshes.erase(node->meshes.begin() + 1, node->meshes.end()); + } + } +} + /* * Export the root node of the node hierarchy. * Calls ExportNode for all children. diff --git a/code/glTF2Exporter.h b/code/glTF2Exporter.h index 420c2afa6..3aed35ae6 100644 --- a/code/glTF2Exporter.h +++ b/code/glTF2Exporter.h @@ -120,6 +120,7 @@ namespace Assimp void ExportMetadata(); void ExportMaterials(); void ExportMeshes(); + void MergeMeshes(); unsigned int ExportNodeHierarchy(const aiNode* n); unsigned int ExportNode(const aiNode* node, glTF2::Ref& parent); void ExportScene(); From 814e8b3f8eb2f10263378f0a840f3d3d55a98916 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 18 Sep 2017 10:20:02 -0400 Subject: [PATCH 175/305] Formatting --- code/glTF2Asset.inl | 3 --- 1 file changed, 3 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 0a3e2b051..47904b38f 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -934,11 +934,8 @@ inline void Node::Read(Value& obj, Asset& r) } if (Value* mesh = FindUInt(obj, "mesh")) { - //unsigned numMeshes = (unsigned)meshes->Size(); unsigned numMeshes = 1; - //std::vector meshList; - this->meshes.reserve(numMeshes); Ref meshRef = r.meshes.Retrieve((*mesh).GetUint()); From 2efd2cdef88f498eb68c143902804d41e8ebcc50 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 18 Sep 2017 10:43:05 -0400 Subject: [PATCH 176/305] tweaks to primitive merging logic; comments + formatting --- code/glTF2Exporter.cpp | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index f63c8e89c..888721c41 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -744,6 +744,7 @@ void glTF2Exporter::ExportMeshes() } } +//merges a node's multiple meshes (with one primitive each) into one mesh with multiple primitives void glTF2Exporter::MergeMeshes() { for (unsigned int n = 0; n < mAsset->nodes.Size(); ++n) { @@ -751,16 +752,34 @@ void glTF2Exporter::MergeMeshes() unsigned int nMeshes = node->meshes.size(); - if (nMeshes) { + //skip if it's 1 or less meshes per node + if (nMeshes > 1) { Ref firstMesh = node->meshes.at(0); + Mesh::Primitive firstPrimitive = firstMesh->primitives.at(0); - for (unsigned int m = 1; m < nMeshes; ++m) { + //loop backwards to allow easy removal of a mesh from a node once it's merged + for (unsigned int m = nMeshes - 1; m >= 1; --m) { Ref mesh = node->meshes.at(m); - Mesh::Primitive primitive = mesh->primitives.at(0); - firstMesh->primitives.push_back(primitive); + bool primitivesPushed = false; + + for (unsigned int p = 0; p < mesh->primitives.size(); ++p) { + Mesh::Primitive primitive = mesh->primitives.at(p); + + if (firstPrimitive.mode == primitive.mode) { + firstMesh->primitives.push_back(primitive); + primitivesPushed = true; + } + } + + if (primitivesPushed) { + //remove the merged meshes from the node + node->meshes.erase(node->meshes.begin() + m); + } } - node->meshes.erase(node->meshes.begin() + 1, node->meshes.end()); + //since we were looping backwards, reverse the order of merged primitives to their original order + std::reverse(firstMesh->primitives.begin() + 1, firstMesh->primitives.end()); + } } } From 8743d28ec54df8d303097ebd0c7741f6e882c21f Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 18 Sep 2017 12:16:30 -0400 Subject: [PATCH 177/305] SImplify mesh merging code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My assumption that primitives of different types (modes) can’t be in the same mesh was incorrect. --- code/glTF2Exporter.cpp | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 888721c41..576e8c723 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -755,31 +755,18 @@ void glTF2Exporter::MergeMeshes() //skip if it's 1 or less meshes per node if (nMeshes > 1) { Ref firstMesh = node->meshes.at(0); - Mesh::Primitive firstPrimitive = firstMesh->primitives.at(0); //loop backwards to allow easy removal of a mesh from a node once it's merged for (unsigned int m = nMeshes - 1; m >= 1; --m) { Ref mesh = node->meshes.at(m); - bool primitivesPushed = false; - for (unsigned int p = 0; p < mesh->primitives.size(); ++p) { - Mesh::Primitive primitive = mesh->primitives.at(p); + firstMesh->primitives.insert(firstMesh->primitives.end(), mesh->primitives.begin(), mesh->primitives.end()); - if (firstPrimitive.mode == primitive.mode) { - firstMesh->primitives.push_back(primitive); - primitivesPushed = true; - } - } - - if (primitivesPushed) { - //remove the merged meshes from the node - node->meshes.erase(node->meshes.begin() + m); - } + node->meshes.erase(node->meshes.begin() + m); } //since we were looping backwards, reverse the order of merged primitives to their original order std::reverse(firstMesh->primitives.begin() + 1, firstMesh->primitives.end()); - } } } From de0bf2ea966905c4fdc70257c68b60a0e2d3aa22 Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 18 Sep 2017 12:19:55 -0400 Subject: [PATCH 178/305] Fix alphaMode storage and reading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit alphaMode is now converted from a std::string to an aiString and back to std::string, since aiString is easier to store and retrieve from aiMaterial properties than std::string Fixes issues of alphaMode being written out as `\fOPAQUE\0\0\0\0\0\0…` --- code/glTF2Exporter.cpp | 17 ++++------------- code/glTF2Importer.cpp | 4 +++- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index a3940e9ea..aa7d5224a 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -424,20 +424,11 @@ void glTF2Exporter::ExportMaterials() mat->Get(AI_MATKEY_TWOSIDED, m->doubleSided); mat->Get(AI_MATKEY_GLTF_ALPHACUTOFF, m->alphaCutoff); - bool foundAlphaMode = false; - for (size_t i = 0; i < mat->mNumProperties; ++i) { - aiMaterialProperty *prop = mat->mProperties[i]; - if (prop->mKey != aiString("$mat.gltf.alphaMode")) - continue; + aiString alphaMode; - std::string alphaMode; - for (size_t c = 0; c < prop->mDataLength; ++c) - alphaMode += prop->mData[c]; - m->alphaMode = alphaMode; - foundAlphaMode = true; - } - - if (!foundAlphaMode) { + if (mat->Get(AI_MATKEY_GLTF_ALPHAMODE, alphaMode) == AI_SUCCESS) { + m->alphaMode = alphaMode.C_Str(); + } else { float opacity; if (mat->Get(AI_MATKEY_OPACITY, opacity) == AI_SUCCESS) { diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index 6de482981..0c0b13bdb 100644 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -239,7 +239,9 @@ void glTF2Importer::ImportMaterials(glTF2::Asset& r) SetMaterialColorProperty(r, mat.emissiveFactor, aimat, AI_MATKEY_COLOR_EMISSIVE); aimat->AddProperty(&mat.doubleSided, 1, AI_MATKEY_TWOSIDED); - aimat->AddProperty(&mat.alphaMode, 1, AI_MATKEY_GLTF_ALPHAMODE); + + aiString alphaMode(mat.alphaMode); + aimat->AddProperty(&alphaMode, AI_MATKEY_GLTF_ALPHAMODE); aimat->AddProperty(&mat.alphaCutoff, 1, AI_MATKEY_GLTF_ALPHACUTOFF); //pbrSpecularGlossiness From 798542d7bde204b8bf896d260885be76065d40aa Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 18 Sep 2017 14:48:07 -0400 Subject: [PATCH 179/305] Formatting --- code/glTF2Exporter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 576e8c723..fb94e1c95 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -479,14 +479,14 @@ bool FindMeshNode(Ref& nodeIn, Ref& meshNode, std::string meshID) { for (unsigned int i = 0; i < nodeIn->meshes.size(); ++i) { if (meshID.compare(nodeIn->meshes[i]->id) == 0) { - meshNode = nodeIn; - return true; + meshNode = nodeIn; + return true; } } for (unsigned int i = 0; i < nodeIn->children.size(); ++i) { if(FindMeshNode(nodeIn->children[i], meshNode, meshID)) { - return true; + return true; } } From 3e8955faf58a04531d3c3d4124821b3da2ced7ef Mon Sep 17 00:00:00 2001 From: Daniel Hritzkiv Date: Mon, 18 Sep 2017 18:16:48 -0400 Subject: [PATCH 180/305] =?UTF-8?q?Don=E2=80=99t=20ignore=20rgba(1,1,1,1)?= =?UTF-8?q?=20color=20properties?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do not ignore rgba(1,1,1,1) material properties when importing glTF1. While a white diffuse color may be the default value for a default assimp material, `1,1,1,1` is a very explicit color value for ambient/specular/emissive color properties. Closes #1434 --- code/glTFImporter.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/code/glTFImporter.cpp b/code/glTFImporter.cpp index 1b5c953be..a1d8e0cd0 100644 --- a/code/glTFImporter.cpp +++ b/code/glTFImporter.cpp @@ -174,9 +174,7 @@ inline void SetMaterialColorProperty(std::vector& embeddedTexIdxs, Asset& r else { aiColor4D col; CopyValue(prop.color, col); - if (col.r != 1.f || col.g != 1.f || col.b != 1.f || col.a != 1.f) { - mat->AddProperty(&col, 1, pKey, type, idx); - } + mat->AddProperty(&col, 1, pKey, type, idx); } } @@ -672,7 +670,7 @@ void glTFImporter::InternReadFile(const std::string& pFile, aiScene* pScene, IOS //pScene->mFlags |= AI_SCENE_FLAGS_NON_VERBOSE_FORMAT; MakeVerboseFormatProcess process; process.Execute(pScene); - + if (pScene->mNumMeshes == 0) { pScene->mFlags |= AI_SCENE_FLAGS_INCOMPLETE; From af9596674db63571359f163bc72125667538ad8b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 19 Sep 2017 00:31:41 +0200 Subject: [PATCH 181/305] FBX: add missing inversion of postrotation matrix for fbx. --- code/FBXConverter.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index f53ae0218..8d85bb8ad 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -1065,6 +1065,10 @@ void Converter::GenerateTransformationNodeChain( const Model& model, continue; } + if ( comp == TransformationComp_PostRotation ) { + chain[ i ] = chain[ i ].Inverse(); + } + aiNode* nd = new aiNode(); output_nodes.push_back( nd ); From 059a32654e660ac9dcfd9a31a8436fce6042c66f Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 24 Sep 2017 19:29:43 +1000 Subject: [PATCH 182/305] Addressed warnings generated on MSVC across x86 and x64. --- code/MMDImporter.cpp | 2 +- code/glTF2Exporter.cpp | 2 +- samples/SimpleOpenGL/CMakeLists.txt | 5 +++++ samples/SimpleTexturedOpenGL/CMakeLists.txt | 5 +++++ tools/assimp_view/stdafx.h | 1 - 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/code/MMDImporter.cpp b/code/MMDImporter.cpp index 84797cba2..c40978c0e 100644 --- a/code/MMDImporter.cpp +++ b/code/MMDImporter.cpp @@ -118,7 +118,7 @@ void MMDImporter::InternReadFile(const std::string &file, aiScene *pScene, // Get the file-size and validate it, throwing an exception when fails fileStream.seekg(0, fileStream.end); - size_t fileSize = fileStream.tellg(); + size_t fileSize = static_cast(fileStream.tellg()); fileStream.seekg(0, fileStream.beg); if (fileSize < sizeof(pmx::PmxModel)) { diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 12642a961..4ed8f20ff 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -741,7 +741,7 @@ void glTF2Exporter::MergeMeshes() for (unsigned int n = 0; n < mAsset->nodes.Size(); ++n) { Ref node = mAsset->nodes.Get(n); - unsigned int nMeshes = node->meshes.size(); + unsigned int nMeshes = static_cast(node->meshes.size()); //skip if it's 1 or less meshes per node if (nMeshes > 1) { diff --git a/samples/SimpleOpenGL/CMakeLists.txt b/samples/SimpleOpenGL/CMakeLists.txt index 52c994b63..455cbd8ca 100644 --- a/samples/SimpleOpenGL/CMakeLists.txt +++ b/samples/SimpleOpenGL/CMakeLists.txt @@ -16,6 +16,11 @@ IF ( NOT GLUT_FOUND ) ENDIF ( MSVC ) ENDIF ( NOT GLUT_FOUND ) +if ( MSVC ) + ADD_DEFINITIONS( -D_SCL_SECURE_NO_WARNINGS ) + ADD_DEFINITIONS( -D_CRT_SECURE_NO_WARNINGS ) +endif ( MSVC ) + INCLUDE_DIRECTORIES( ${Assimp_SOURCE_DIR}/include ${Assimp_SOURCE_DIR}/code diff --git a/samples/SimpleTexturedOpenGL/CMakeLists.txt b/samples/SimpleTexturedOpenGL/CMakeLists.txt index 6c34acda5..1b206af50 100644 --- a/samples/SimpleTexturedOpenGL/CMakeLists.txt +++ b/samples/SimpleTexturedOpenGL/CMakeLists.txt @@ -11,6 +11,11 @@ IF ( NOT GLUT_FOUND ) ENDIF ( MSVC ) ENDIF ( NOT GLUT_FOUND ) +if ( MSVC ) + ADD_DEFINITIONS( -D_SCL_SECURE_NO_WARNINGS ) + ADD_DEFINITIONS( -D_CRT_SECURE_NO_WARNINGS ) +endif ( MSVC ) + INCLUDE_DIRECTORIES( ${Assimp_SOURCE_DIR}/include ${Assimp_SOURCE_DIR}/code diff --git a/tools/assimp_view/stdafx.h b/tools/assimp_view/stdafx.h index 3c78d385c..35104d4b0 100644 --- a/tools/assimp_view/stdafx.h +++ b/tools/assimp_view/stdafx.h @@ -23,7 +23,6 @@ #define _WIN32_IE 0x0600 // Ändern Sie dies in den geeigneten Wert für andere Versionen von IE. #endif -#define WIN32_LEAN_AND_MEAN // Selten verwendete Teile der Windows-Header nicht einbinden. // Windows-Headerdateien: #include From 539410d033f0606e444601ab2224bb70ce52c9a5 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 24 Sep 2017 19:31:04 +1000 Subject: [PATCH 183/305] Fixed an error when compiling samples under MSVC that was caused by assuming including windows.h would pull in shellapi.h --- .../SimpleTexturedOpenGL/src/model_loading.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/SimpleTexturedOpenGL/SimpleTexturedOpenGL/src/model_loading.cpp b/samples/SimpleTexturedOpenGL/SimpleTexturedOpenGL/src/model_loading.cpp index 083650f96..df05b56c4 100644 --- a/samples/SimpleTexturedOpenGL/SimpleTexturedOpenGL/src/model_loading.cpp +++ b/samples/SimpleTexturedOpenGL/SimpleTexturedOpenGL/src/model_loading.cpp @@ -13,6 +13,7 @@ // http://nehe.gamedev.net/ // ---------------------------------------------------------------------------- #include +#include #include #include #include @@ -666,7 +667,7 @@ BOOL CreateGLWindow(const char* title, int width, int height, int bits, bool ful PFD_SUPPORT_OPENGL | // Format Must Support OpenGL PFD_DOUBLEBUFFER, // Must Support Double Buffering PFD_TYPE_RGBA, // Request An RGBA Format - bits, // Select Our Color Depth + BYTE(bits), // Select Our Color Depth 0, 0, 0, 0, 0, 0, // Color Bits Ignored 0, // No Alpha Buffer 0, // Shift Bit Ignored From 8dabd76e03016b9efdffbfce2ef2e7d1fa9db403 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 24 Sep 2017 21:19:03 +1000 Subject: [PATCH 184/305] Fixed a warning caused by aiVector3D appearing in a packed struct, causing it to fail to pack as requested. --- code/MDLFileData.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/MDLFileData.h b/code/MDLFileData.h index 2afea8a95..3ef58ca5a 100644 --- a/code/MDLFileData.h +++ b/code/MDLFileData.h @@ -126,16 +126,16 @@ struct Header { int32_t version; //! scale factors for each axis - aiVector3D scale; + ai_real scale[3]; //! translation factors for each axis - aiVector3D translate; + ai_real translate[3]; //! bounding radius of the mesh float boundingradius; //! Position of the viewer's exe. Ignored - aiVector3D vEyePos; + ai_real vEyePos[3]; //! Number of textures int32_t num_skins; From 79a51651068b5b5bef4d7b609a7301d9f6790b89 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 24 Sep 2017 21:46:15 +1000 Subject: [PATCH 185/305] Fixed unused variable warning by replacing them with descriptive comments --- code/FBXBinaryTokenizer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/FBXBinaryTokenizer.cpp b/code/FBXBinaryTokenizer.cpp index 46e85d888..ede32d7b7 100644 --- a/code/FBXBinaryTokenizer.cpp +++ b/code/FBXBinaryTokenizer.cpp @@ -439,11 +439,11 @@ void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int le } const char* cursor = input + 18; - const uint8_t unknown_1 = ReadByte(input, cursor, input + length); - const uint8_t unknown_2 = ReadByte(input, cursor, input + length); - const uint8_t unknown_3 = ReadByte(input, cursor, input + length); - const uint8_t unknown_4 = ReadByte(input, cursor, input + length); - const uint8_t unknown_5 = ReadByte(input, cursor, input + length); + /*Result ignored*/ ReadByte(input, cursor, input + length); + /*Result ignored*/ ReadByte(input, cursor, input + length); + /*Result ignored*/ ReadByte(input, cursor, input + length); + /*Result ignored*/ ReadByte(input, cursor, input + length); + /*Result ignored*/ ReadByte(input, cursor, input + length); const uint32_t version = ReadWord(input, cursor, input + length); const bool is64bits = version >= 7500; while (cursor < input + length) From 7e91ac34436d8f0708e1b43cbe5b4f2e3885e9a8 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Mon, 25 Sep 2017 20:22:06 +1000 Subject: [PATCH 186/305] Suppressed warning on gcc caused by the 'visibility' attribute being ignored on types. --- include/assimp/scene.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/assimp/scene.h b/include/assimp/scene.h index 4d027456c..342c316d6 100644 --- a/include/assimp/scene.h +++ b/include/assimp/scene.h @@ -60,6 +60,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. extern "C" { #endif +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wattributes" +#endif + // ------------------------------------------------------------------------------- /** * A node in the imported hierarchy. @@ -163,6 +168,9 @@ struct ASSIMP_API aiNode #endif // __cplusplus }; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif // ------------------------------------------------------------------------------- /** From f6fc5a7a117ea41e43b0bdb1426e9b319924d8fd Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Mon, 25 Sep 2017 22:07:01 +1000 Subject: [PATCH 187/305] Changed the FileSizeTest to not rely on tmpnam to eliminate warning on gcc. --- test/unit/utDefaultIOStream.cpp | 47 ++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/test/unit/utDefaultIOStream.cpp b/test/unit/utDefaultIOStream.cpp index 9c1beb193..9ef2bad86 100644 --- a/test/unit/utDefaultIOStream.cpp +++ b/test/unit/utDefaultIOStream.cpp @@ -39,6 +39,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------*/ #include #include "TestIOStream.h" +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define MAKE_TEMP_FILE_NAME mktemp +#elif defined(_MSC_VER) +#include +#define MAKE_TEMP_FILE_NAME _mktemp +#endif using namespace ::Assimp; @@ -46,16 +56,33 @@ class utDefaultIOStream : public ::testing::Test { // empty }; -TEST_F( utDefaultIOStream, FileSizeTest ) { - char buffer[ L_tmpnam ]; - tmpnam( buffer ); - std::FILE *fs( std::fopen( buffer, "w+" ) ); - size_t written( std::fwrite( buffer, 1, sizeof( char ) * L_tmpnam, fs ) ); - EXPECT_NE( 0U, written ); - std::fflush( fs ); - TestDefaultIOStream myStream( fs, buffer ); + +const char data[]{"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Qui\ +sque luctus sem diam, ut eleifend arcu auctor eu. Vestibulum id est vel nulla l\ +obortis malesuada ut sed turpis. Nulla a volutpat tortor. Nunc vestibulum portt\ +itor sapien ornare sagittis volutpat."}; + +TEST_F( utDefaultIOStream, FileSizeTest ) { + const auto dataSize = sizeof(data); + const auto dataCount = dataSize / sizeof(*data); + + char fpath[] = { "./rndfp.XXXXXX" }; + auto tmplate = MAKE_TEMP_FILE_NAME(fpath); + ASSERT_NE(tmplate, nullptr); + + //char buffer[ L_tmpnam ]; + //tmpnam( buffer ); + auto *fs = std::fopen(fpath, "w+" ); + ASSERT_NE(fs, nullptr); + auto written = std::fwrite(data, sizeof(*data), dataCount, fs ); + EXPECT_NE( 0U, written ); + auto vflush = std::fflush( fs ); + ASSERT_EQ(vflush, 0); + + TestDefaultIOStream myStream( fs, fpath); size_t size = myStream.FileSize(); - EXPECT_EQ( size, sizeof( char ) * L_tmpnam ); - remove( buffer ); + EXPECT_EQ( size, dataSize); + remove(fpath); + } From 4360267cb22daec5e7e35d4b96d56dac4879feb5 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Mon, 25 Sep 2017 22:58:47 +1000 Subject: [PATCH 188/305] Replaced flakey macros with specific functions to serve the purpose --- test/unit/utDefaultIOStream.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/unit/utDefaultIOStream.cpp b/test/unit/utDefaultIOStream.cpp index 9ef2bad86..9335007ba 100644 --- a/test/unit/utDefaultIOStream.cpp +++ b/test/unit/utDefaultIOStream.cpp @@ -44,10 +44,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #if defined(__GNUC__) || defined(__clang__) -#define MAKE_TEMP_FILE_NAME mktemp +#define TMP_PATH "/tmp/" +void MakeTmpFilePath(char* tmplate) +{ + auto err = mkstemp(tmplate); + ASSERT_NE(err, -1); +} #elif defined(_MSC_VER) #include -#define MAKE_TEMP_FILE_NAME _mktemp +#define TMP_PATH "./" +void MakeTmpFilePath(char* tmplate) +{ + auto pathtemplate = _mktemp(tmplate); + ASSERT_NE(pathtemplate, nullptr); +} #endif using namespace ::Assimp; @@ -67,12 +77,9 @@ TEST_F( utDefaultIOStream, FileSizeTest ) { const auto dataSize = sizeof(data); const auto dataCount = dataSize / sizeof(*data); - char fpath[] = { "./rndfp.XXXXXX" }; - auto tmplate = MAKE_TEMP_FILE_NAME(fpath); - ASSERT_NE(tmplate, nullptr); + char fpath[] = { TMP_PATH"rndfp.XXXXXX" }; + MakeTmpFilePath(fpath); - //char buffer[ L_tmpnam ]; - //tmpnam( buffer ); auto *fs = std::fopen(fpath, "w+" ); ASSERT_NE(fs, nullptr); auto written = std::fwrite(data, sizeof(*data), dataCount, fs ); From e2ab3e0d29511d60529fd0ff22bd85be896902fe Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 26 Sep 2017 21:32:11 +1000 Subject: [PATCH 189/305] Changed the method by which temporary files are created for unit the FileSizeTest. Will apply to other tests next. --- test/unit/utDefaultIOStream.cpp | 70 +++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/test/unit/utDefaultIOStream.cpp b/test/unit/utDefaultIOStream.cpp index 9335007ba..c5f5ae760 100644 --- a/test/unit/utDefaultIOStream.cpp +++ b/test/unit/utDefaultIOStream.cpp @@ -43,30 +43,42 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#if defined(__GNUC__) || defined(__clang__) -#define TMP_PATH "/tmp/" -void MakeTmpFilePath(char* tmplate) -{ - auto err = mkstemp(tmplate); - ASSERT_NE(err, -1); -} -#elif defined(_MSC_VER) -#include -#define TMP_PATH "./" -void MakeTmpFilePath(char* tmplate) -{ - auto pathtemplate = _mktemp(tmplate); - ASSERT_NE(pathtemplate, nullptr); -} -#endif - using namespace ::Assimp; class utDefaultIOStream : public ::testing::Test { // empty }; - +#if defined(__GNUC__) || defined(__clang__) +#define TMP_PATH "/tmp/" +FILE* MakeTmpFilePath(char* tmplate) +{ + auto fd = mkstemp(tmplate); + EXPECT_NE(-1, fd); + if(fd == -1) + { + return nullptr; + } + auto fs = fdopen(fd, "w+"); + EXPECT_NE(nullptr, fs); + return fs; +} +#elif defined(_MSC_VER) +#include +#define TMP_PATH "./" +FILE* MakeTmpFilePath(char* tmplate) +{ + auto pathtemplate = _mktemp(tmplate); + EXPECT_NE(pathtemplate, nullptr); + if(pathtemplate == nullptr) + { + return nullptr; + } + auto* fs = std::fopen(pathtemplate, "w+"); + EXPECT_NE(fs, nullptr); + return fs; +} +#endif const char data[]{"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Qui\ sque luctus sem diam, ut eleifend arcu auctor eu. Vestibulum id est vel nulla l\ @@ -78,18 +90,18 @@ TEST_F( utDefaultIOStream, FileSizeTest ) { const auto dataCount = dataSize / sizeof(*data); char fpath[] = { TMP_PATH"rndfp.XXXXXX" }; - MakeTmpFilePath(fpath); + auto* fs = MakeTmpFilePath(fpath); + ASSERT_NE(nullptr, fs); + { + auto written = std::fwrite(data, sizeof(*data), dataCount, fs ); + EXPECT_NE( 0U, written ); - auto *fs = std::fopen(fpath, "w+" ); - ASSERT_NE(fs, nullptr); - auto written = std::fwrite(data, sizeof(*data), dataCount, fs ); - EXPECT_NE( 0U, written ); - auto vflush = std::fflush( fs ); - ASSERT_EQ(vflush, 0); + auto vflush = std::fflush( fs ); + ASSERT_EQ(vflush, 0); - TestDefaultIOStream myStream( fs, fpath); - size_t size = myStream.FileSize(); - EXPECT_EQ( size, dataSize); + TestDefaultIOStream myStream( fs, fpath); + size_t size = myStream.FileSize(); + EXPECT_EQ( size, dataSize); + } remove(fpath); - } From 980d2b0eee6981625d92bc01dfd24ab06f3068db Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 26 Sep 2017 22:08:52 +1000 Subject: [PATCH 190/305] Added a header to hold the file generation code for unit testing purposes. --- test/unit/UnitTestFileGenerator.h | 76 +++++++++++++++++++++++++++++++ test/unit/utDefaultIOStream.cpp | 32 +------------ 2 files changed, 77 insertions(+), 31 deletions(-) create mode 100644 test/unit/UnitTestFileGenerator.h diff --git a/test/unit/UnitTestFileGenerator.h b/test/unit/UnitTestFileGenerator.h new file mode 100644 index 000000000..04f5fd409 --- /dev/null +++ b/test/unit/UnitTestFileGenerator.h @@ -0,0 +1,76 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +#pragma once + +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define TMP_PATH "/tmp/" +inline FILE* MakeTmpFilePath(char* tmplate) +{ + auto fd = mkstemp(tmplate); + EXPECT_NE(-1, fd); + if(fd == -1) + { + return nullptr; + } + auto fs = fdopen(fd, "w+"); + EXPECT_NE(nullptr, fs); + return fs; +} +#elif defined(_MSC_VER) +#include +#define TMP_PATH "./" +inline FILE* MakeTmpFilePath(char* tmplate) +{ + auto pathtemplate = _mktemp(tmplate); + EXPECT_NE(pathtemplate, nullptr); + if(pathtemplate == nullptr) + { + return nullptr; + } + auto* fs = std::fopen(pathtemplate, "w+"); + EXPECT_NE(fs, nullptr); + return fs; +} +#endif diff --git a/test/unit/utDefaultIOStream.cpp b/test/unit/utDefaultIOStream.cpp index c5f5ae760..1d78fb616 100644 --- a/test/unit/utDefaultIOStream.cpp +++ b/test/unit/utDefaultIOStream.cpp @@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------*/ #include #include "TestIOStream.h" +#include "UnitTestFileGenerator.h" #include #include #include @@ -49,37 +50,6 @@ class utDefaultIOStream : public ::testing::Test { // empty }; -#if defined(__GNUC__) || defined(__clang__) -#define TMP_PATH "/tmp/" -FILE* MakeTmpFilePath(char* tmplate) -{ - auto fd = mkstemp(tmplate); - EXPECT_NE(-1, fd); - if(fd == -1) - { - return nullptr; - } - auto fs = fdopen(fd, "w+"); - EXPECT_NE(nullptr, fs); - return fs; -} -#elif defined(_MSC_VER) -#include -#define TMP_PATH "./" -FILE* MakeTmpFilePath(char* tmplate) -{ - auto pathtemplate = _mktemp(tmplate); - EXPECT_NE(pathtemplate, nullptr); - if(pathtemplate == nullptr) - { - return nullptr; - } - auto* fs = std::fopen(pathtemplate, "w+"); - EXPECT_NE(fs, nullptr); - return fs; -} -#endif - const char data[]{"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Qui\ sque luctus sem diam, ut eleifend arcu auctor eu. Vestibulum id est vel nulla l\ obortis malesuada ut sed turpis. Nulla a volutpat tortor. Nunc vestibulum portt\ From e77e89c8b7dd6391b300262670b884a407021a3b Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 26 Sep 2017 22:41:20 +1000 Subject: [PATCH 191/305] Improved the naming of temporary file generator function. Replaced use of tmpnam in utIOStreamBuffer with this facility to addresssafety warning. --- test/unit/UnitTestFileGenerator.h | 4 +-- test/unit/utDefaultIOStream.cpp | 2 +- test/unit/utIOStreamBuffer.cpp | 56 +++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/test/unit/UnitTestFileGenerator.h b/test/unit/UnitTestFileGenerator.h index 04f5fd409..2e4aede0f 100644 --- a/test/unit/UnitTestFileGenerator.h +++ b/test/unit/UnitTestFileGenerator.h @@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(__GNUC__) || defined(__clang__) #define TMP_PATH "/tmp/" -inline FILE* MakeTmpFilePath(char* tmplate) +inline FILE* MakeTmpFile(char* tmplate) { auto fd = mkstemp(tmplate); EXPECT_NE(-1, fd); @@ -61,7 +61,7 @@ inline FILE* MakeTmpFilePath(char* tmplate) #elif defined(_MSC_VER) #include #define TMP_PATH "./" -inline FILE* MakeTmpFilePath(char* tmplate) +inline FILE* MakeTmpFile(char* tmplate) { auto pathtemplate = _mktemp(tmplate); EXPECT_NE(pathtemplate, nullptr); diff --git a/test/unit/utDefaultIOStream.cpp b/test/unit/utDefaultIOStream.cpp index 1d78fb616..c0d82b6dc 100644 --- a/test/unit/utDefaultIOStream.cpp +++ b/test/unit/utDefaultIOStream.cpp @@ -60,7 +60,7 @@ TEST_F( utDefaultIOStream, FileSizeTest ) { const auto dataCount = dataSize / sizeof(*data); char fpath[] = { TMP_PATH"rndfp.XXXXXX" }; - auto* fs = MakeTmpFilePath(fpath); + auto* fs = MakeTmpFile(fpath); ASSERT_NE(nullptr, fs); { auto written = std::fwrite(data, sizeof(*data), dataCount, fs ); diff --git a/test/unit/utIOStreamBuffer.cpp b/test/unit/utIOStreamBuffer.cpp index 0bf69bf00..a1b67da44 100644 --- a/test/unit/utIOStreamBuffer.cpp +++ b/test/unit/utIOStreamBuffer.cpp @@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "UnitTestPCH.h" #include "IOStreamBuffer.h" #include "TestIOStream.h" +#include "UnitTestFileGenerator.h" class IOStreamBufferTest : public ::testing::Test { // empty @@ -68,40 +69,59 @@ TEST_F( IOStreamBufferTest, accessCacheSizeTest ) { EXPECT_EQ( 100U, myBuffer2.cacheSize() ); } +const char data[]{"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Qui\ +sque luctus sem diam, ut eleifend arcu auctor eu. Vestibulum id est vel nulla l\ +obortis malesuada ut sed turpis. Nulla a volutpat tortor. Nunc vestibulum portt\ +itor sapien ornare sagittis volutpat."}; + + TEST_F( IOStreamBufferTest, open_close_Test ) { IOStreamBuffer myBuffer; EXPECT_FALSE( myBuffer.open( nullptr ) ); EXPECT_FALSE( myBuffer.close() ); + + const auto dataSize = sizeof(data); + const auto dataCount = dataSize / sizeof(*data); - char buffer[ L_tmpnam ]; - tmpnam( buffer ); - std::FILE *fs( std::fopen( buffer, "w+" ) ); - size_t written( std::fwrite( buffer, 1, sizeof( char ) * L_tmpnam, fs ) ); + char fname[]={ "octest.XXXXXX" }; + auto* fs = MakeTmpFile(fname); + ASSERT_NE(nullptr, fs); + + auto written = std::fwrite( data, sizeof(*data), dataCount, fs ); EXPECT_NE( 0U, written ); std::fflush( fs ); + { + TestDefaultIOStream myStream( fs, fname ); - TestDefaultIOStream myStream( fs, buffer ); - - EXPECT_TRUE( myBuffer.open( &myStream ) ); - EXPECT_FALSE( myBuffer.open( &myStream ) ); - EXPECT_TRUE( myBuffer.close() ); + EXPECT_TRUE( myBuffer.open( &myStream ) ); + EXPECT_FALSE( myBuffer.open( &myStream ) ); + EXPECT_TRUE( myBuffer.close() ); + } + remove(fname); } TEST_F( IOStreamBufferTest, readlineTest ) { - char buffer[ L_tmpnam ]; - tmpnam( buffer ); - std::FILE *fs( std::fopen( buffer, "w+" ) ); - size_t written( std::fwrite( buffer, 1, sizeof( char ) * L_tmpnam, fs ) ); + + const auto dataSize = sizeof(data); + const auto dataCount = dataSize / sizeof(*data); + + char fname[]={ "readlinetest.XXXXXX" }; + auto* fs = MakeTmpFile(fname); + ASSERT_NE(nullptr, fs); + + auto written = std::fwrite( data, sizeof(*data), dataCount, fs ); EXPECT_NE( 0U, written ); std::fflush( fs ); - IOStreamBuffer myBuffer( 26 ); - EXPECT_EQ( 26U, myBuffer.cacheSize() ); + const auto tCacheSize = 26u; - TestDefaultIOStream myStream( fs, buffer ); - size_t size( myStream.FileSize() ); - size_t numBlocks( size / myBuffer.cacheSize() ); + IOStreamBuffer myBuffer( tCacheSize ); + EXPECT_EQ(tCacheSize, myBuffer.cacheSize() ); + + TestDefaultIOStream myStream( fs, fname ); + auto size = myStream.FileSize(); + auto numBlocks = size / myBuffer.cacheSize(); if ( size % myBuffer.cacheSize() > 0 ) { numBlocks++; } From febd611d48c064f74341b42906229bd1a57a8e39 Mon Sep 17 00:00:00 2001 From: Josh Faust Date: Wed, 27 Sep 2017 18:41:35 -0700 Subject: [PATCH 192/305] Fix glTF2::Asset::FindUniqueID() when the input string is >= 256 chars --- code/glTF2Asset.inl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 47904b38f..264a32d4d 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -1118,11 +1118,12 @@ inline std::string Asset::FindUniqueID(const std::string& str, const char* suffi if (it == mUsedIds.end()) return id; - char buffer[256]; - int offset = ai_snprintf(buffer, sizeof(buffer), "%s_", id.c_str()); + std::vector buffer; + buffer.resize(id.size() + 16); + int offset = ai_snprintf(buffer.data(), buffer.size(), "%s_", id.c_str()); for (int i = 0; it != mUsedIds.end(); ++i) { - ai_snprintf(buffer + offset, sizeof(buffer) - offset, "%d", i); - id = buffer; + ai_snprintf(buffer.data() + offset, buffer.size() - offset, "%d", i); + id = buffer.data(); it = mUsedIds.find(id); } From 12b6895f7b682255f60dcd77dd26a75da63a4697 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Fri, 29 Sep 2017 01:29:11 +1000 Subject: [PATCH 193/305] Replaced unsigned long for the crc table to z_crc_t, to match what is returned by get-crc_table --- contrib/unzip/crypt.h | 2 +- contrib/unzip/unzip.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/unzip/crypt.h b/contrib/unzip/crypt.h index 622f4bc2e..5fd3dcd6c 100644 --- a/contrib/unzip/crypt.h +++ b/contrib/unzip/crypt.h @@ -45,7 +45,7 @@ static int decrypt_byte(unsigned long* pkeys, const unsigned long* pcrc_32_tab) /*********************************************************************** * Update the encryption keys with the next byte of plain text */ -static int update_keys(unsigned long* pkeys,const unsigned long* pcrc_32_tab,int c) +static int update_keys(unsigned long* pkeys,const z_crc_t* pcrc_32_tab,int c) { (*(pkeys+0)) = CRC32((*(pkeys+0)), c); (*(pkeys+1)) += (*(pkeys+0)) & 0xff; diff --git a/contrib/unzip/unzip.c b/contrib/unzip/unzip.c index 9ad4766d8..e9ad1ff99 100644 --- a/contrib/unzip/unzip.c +++ b/contrib/unzip/unzip.c @@ -147,7 +147,7 @@ typedef struct int encrypted; # ifndef NOUNCRYPT unsigned long keys[3]; /* keys defining the pseudo-random sequence */ - const unsigned long* pcrc_32_tab; + const z_crc_t* pcrc_32_tab; # endif } unz_s; From 4feac1b1f93c8f12df65b1592d64b06e6a048fc6 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Fri, 29 Sep 2017 01:40:05 +1000 Subject: [PATCH 194/305] Changed a couple more function interfaces to correct the input type. --- contrib/unzip/crypt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/unzip/crypt.h b/contrib/unzip/crypt.h index 5fd3dcd6c..2aa99c284 100644 --- a/contrib/unzip/crypt.h +++ b/contrib/unzip/crypt.h @@ -32,7 +32,7 @@ /*********************************************************************** * Return the next byte in the pseudo-random sequence */ -static int decrypt_byte(unsigned long* pkeys, const unsigned long* pcrc_32_tab) +static int decrypt_byte(unsigned long* pkeys, const z_crc_t* pcrc_32_tab) { unsigned temp; /* POTENTIAL BUG: temp*(temp^1) may overflow in an * unpredictable manner on 16-bit systems; not a problem @@ -62,7 +62,7 @@ static int update_keys(unsigned long* pkeys,const z_crc_t* pcrc_32_tab,int c) * Initialize the encryption keys and the random header according to * the given password. */ -static void init_keys(const char* passwd,unsigned long* pkeys,const unsigned long* pcrc_32_tab) +static void init_keys(const char* passwd,unsigned long* pkeys,const z_crc_t* pcrc_32_tab) { *(pkeys+0) = 305419896L; *(pkeys+1) = 591751049L; From c42589460d94d0e6334fbb6791309bd1218f5913 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 29 Sep 2017 21:58:58 +0200 Subject: [PATCH 195/305] closes https://github.com/assimp/assimp/issues/1459: fix out-of-boundary access error --- code/SIBImporter.cpp | 53 +++++++++++++++++++------------------------- code/StreamReader.h | 8 ++----- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/code/SIBImporter.cpp b/code/SIBImporter.cpp index 4ae2ab7d4..bdda3c677 100644 --- a/code/SIBImporter.cpp +++ b/code/SIBImporter.cpp @@ -176,36 +176,29 @@ static void UnknownChunk(StreamReaderLE* stream, const SIBChunk& chunk) } // Reads a UTF-16LE string and returns it at UTF-8. -static aiString ReadString(StreamReaderLE* stream, uint32_t numWChars) -{ - if ( 0 == numWChars ) { +static aiString ReadString(StreamReaderLE *stream, uint32_t numWChars) { + if ( nullptr == stream || 0 == numWChars ) { static const aiString empty; return empty; } + // Allocate buffers (max expansion is 1 byte -> 4 bytes for UTF-8) - //UTF16* temp = new UTF16[numWChars]; std::vector str; - str.reserve(numWChars * 4 + 1); - //unsigned char* str = new unsigned char[numWChars * 4 + 1]; - uint16_t *temp = new uint16_t[numWChars]; - for (uint32_t n=0;nGetU2(); + str.reserve( numWChars * 4 + 1 ); + uint16_t *temp = new uint16_t[ numWChars ]; + for ( uint32_t n = 0; n < numWChars; ++n ) { + temp[ n ] = stream->GetU2(); + } // Convert it and NUL-terminate. - //const UTF16 *start = temp, *end = temp + numWChars; + const uint16_t *start( temp ), *end( temp + numWChars ); + utf8::utf16to8( start, end, back_inserter( str ) ); + str[ str.size() - 1 ] = '\0'; - const uint16_t *start = temp, *end = temp + numWChars; - utf8::utf16to8(start, end, back_inserter(str)); - - //UTF8 *dest = str, *limit = str + numWChars*4; - //ConvertUTF16toUTF8(&start, end, &dest, limit, lenientConversion); - //*dest = '\0'; - - str[str.size()] = '\0'; // Return the final string. aiString result = aiString((const char *)&str[0]); - //delete[] str; delete[] temp; + return result; } @@ -223,26 +216,26 @@ SIBImporter::~SIBImporter() { // ------------------------------------------------------------------------------------------------ // Returns whether the class can handle the format of the given file. -bool SIBImporter::CanRead( const std::string& pFile, IOSystem* /*pIOHandler*/, bool /*checkSig*/) const -{ +bool SIBImporter::CanRead( const std::string& pFile, IOSystem* /*pIOHandler*/, bool /*checkSig*/) const { return SimpleExtensionCheck(pFile, "sib"); } // ------------------------------------------------------------------------------------------------ -const aiImporterDesc* SIBImporter::GetInfo () const -{ +const aiImporterDesc* SIBImporter::GetInfo () const { return &desc; } // ------------------------------------------------------------------------------------------------ -static void ReadVerts(SIBMesh* mesh, StreamReaderLE* stream, uint32_t count) -{ - mesh->pos.resize(count); +static void ReadVerts(SIBMesh* mesh, StreamReaderLE* stream, uint32_t count) { + if ( nullptr == mesh || nullptr == stream ) { + return; + } - for (uint32_t n=0;npos[n].x = stream->GetF4(); - mesh->pos[n].y = stream->GetF4(); - mesh->pos[n].z = stream->GetF4(); + mesh->pos.resize(count); + for ( uint32_t n=0; npos[ n ].x = stream->GetF4(); + mesh->pos[ n ].y = stream->GetF4(); + mesh->pos[ n ].z = stream->GetF4(); } } diff --git a/code/StreamReader.h b/code/StreamReader.h index 494aed146..6220de9a8 100644 --- a/code/StreamReader.h +++ b/code/StreamReader.h @@ -291,15 +291,11 @@ private: throw DeadlyImportError("End of file or stream limit was reached"); } -///*#ifdef __arm__ T f; ::memcpy (&f, current, sizeof(T)); -//#else*/ -// T f = *((const T*)current); -//#endif - Intern :: Getter() (&f,le); - + Intern::Getter() (&f,le); current += sizeof(T); + return f; } From 90330712374c20b870cbf7915ace158fe0d069bb Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 30 Sep 2017 09:37:34 +0200 Subject: [PATCH 196/305] Obj: rename attribute from exporter. --- code/ObjExporter.cpp | 44 +++++++++++++++++++++++++------------------- code/ObjExporter.h | 6 +++--- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/code/ObjExporter.cpp b/code/ObjExporter.cpp index 14035c262..cc9d2f31c 100644 --- a/code/ObjExporter.cpp +++ b/code/ObjExporter.cpp @@ -91,11 +91,11 @@ ObjExporter::ObjExporter(const char* _filename, const aiScene* pScene) , vn() , vt() , vc() -, vpMap() -, vnMap() -, vtMap() -, vcMap() -, meshes() +, mVpMap() +, mVnMap() +, mVtMap() +, mVcMap() +, mMeshes() , endl("\n") { // make sure that all formatting happens using the standard, C locale and not the user's current locale const std::locale& l = std::locale("C"); @@ -137,15 +137,21 @@ std::string ObjExporter::GetMaterialLibFileName() { } // ------------------------------------------------------------------------------------------------ -void ObjExporter :: WriteHeader(std::ostringstream& out) { +void ObjExporter::WriteHeader(std::ostringstream& out) { out << "# File produced by Open Asset Import Library (http://www.assimp.sf.net)" << endl; - out << "# (assimp v" << aiGetVersionMajor() << '.' << aiGetVersionMinor() << '.' << aiGetVersionRevision() << ")" << endl << endl; + out << "# (assimp v" << aiGetVersionMajor() << '.' << aiGetVersionMinor() << '.' + << aiGetVersionRevision() << ")" << endl << endl; } // ------------------------------------------------------------------------------------------------ -std::string ObjExporter :: GetMaterialName(unsigned int index) +std::string ObjExporter::GetMaterialName(unsigned int index) { const aiMaterial* const mat = pScene->mMaterials[index]; + if ( nullptr == mat ) { + static const std::string EmptyStr; + return EmptyStr; + } + aiString s; if(AI_SUCCESS == mat->Get(AI_MATKEY_NAME,s)) { return std::string(s.data,s.length); @@ -235,8 +241,8 @@ void ObjExporter::WriteGeometryFile() { AddNode(pScene->mRootNode, mBase); // write vertex positions with colors, if any - vpMap.getVectors( vp ); - vcMap.getColors( vc ); + mVpMap.getVectors( vp ); + mVcMap.getColors( vc ); if ( vc.empty() ) { mOutput << "# " << vp.size() << " vertex positions" << endl; for ( const aiVector3D& v : vp ) { @@ -253,7 +259,7 @@ void ObjExporter::WriteGeometryFile() { mOutput << endl; // write uv coordinates - vtMap.getVectors(vt); + mVtMap.getVectors(vt); mOutput << "# " << vt.size() << " UV coordinates" << endl; for(const aiVector3D& v : vt) { mOutput << "vt " << v.x << " " << v.y << " " << v.z << endl; @@ -261,7 +267,7 @@ void ObjExporter::WriteGeometryFile() { mOutput << endl; // write vertex normals - vnMap.getVectors(vn); + mVnMap.getVectors(vn); mOutput << "# " << vn.size() << " vertex normals" << endl; for(const aiVector3D& v : vn) { mOutput << "vn " << v.x << " " << v.y << " " << v.z << endl; @@ -269,7 +275,7 @@ void ObjExporter::WriteGeometryFile() { mOutput << endl; // now write all mesh instances - for(const MeshInstance& m : meshes) { + for(const MeshInstance& m : mMeshes) { mOutput << "# Mesh \'" << m.name << "\' with " << m.faces.size() << " faces" << endl; if (!m.name.empty()) { mOutput << "g " << m.name << endl; @@ -345,8 +351,8 @@ void ObjExporter::colIndexMap::getColors( std::vector &colors ) { // ------------------------------------------------------------------------------------------------ void ObjExporter::AddMesh(const aiString& name, const aiMesh* m, const aiMatrix4x4& mat) { - meshes.push_back(MeshInstance()); - MeshInstance& mesh = meshes.back(); + mMeshes.push_back(MeshInstance()); + MeshInstance& mesh = mMeshes.back(); mesh.name = std::string(name.data,name.length) + (m->mName.length ? "_" + std::string(m->mName.data,m->mName.length) : ""); mesh.matname = GetMaterialName(m->mMaterialIndex); @@ -373,24 +379,24 @@ void ObjExporter::AddMesh(const aiString& name, const aiMesh* m, const aiMatrix4 const unsigned int idx = f.mIndices[a]; aiVector3D vert = mat * m->mVertices[idx]; - face.indices[a].vp = vpMap.getIndex(vert); + face.indices[a].vp = mVpMap.getIndex(vert); if (m->mNormals) { aiVector3D norm = aiMatrix3x3(mat) * m->mNormals[idx]; - face.indices[a].vn = vnMap.getIndex(norm); + face.indices[a].vn = mVnMap.getIndex(norm); } else { face.indices[a].vn = 0; } if ( nullptr != m->mColors[ 0 ] ) { aiColor4D col4 = m->mColors[ 0 ][ idx ]; - face.indices[ a ].vc = vcMap.getIndex( col4 ); + face.indices[ a ].vc = mVcMap.getIndex( col4 ); } else { face.indices[ a ].vc = 0; } if ( m->mTextureCoords[ 0 ] ) { - face.indices[a].vt = vtMap.getIndex(m->mTextureCoords[0][idx]); + face.indices[a].vt = mVtMap.getIndex(m->mTextureCoords[0][idx]); } else { face.indices[a].vt = 0; } diff --git a/code/ObjExporter.h b/code/ObjExporter.h index 0ba042b0b..fb60402d7 100644 --- a/code/ObjExporter.h +++ b/code/ObjExporter.h @@ -163,9 +163,9 @@ private: void getColors( std::vector &colors ); }; - vecIndexMap vpMap, vnMap, vtMap; - colIndexMap vcMap; - std::vector meshes; + vecIndexMap mVpMap, mVnMap, mVtMap; + colIndexMap mVcMap; + std::vector mMeshes; // this endl() doesn't flush() the stream const std::string endl; From 1c76962c980e42e45eeb0f255162e1dc08740789 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 30 Sep 2017 09:38:13 +0200 Subject: [PATCH 197/305] closes https://github.com/assimp/assimp/issues/1450: use correct name of exporter to gltf2 --- code/Exporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/Exporter.cpp b/code/Exporter.cpp index 7af7d2cf5..1e2d7cbf1 100644 --- a/code/Exporter.cpp +++ b/code/Exporter.cpp @@ -145,7 +145,7 @@ Exporter::ExportFormatEntry gExporters[] = aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_SortByPType), Exporter::ExportFormatEntry( "glb", "GL Transmission Format (binary)", "glb", &ExportSceneGLB, aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_SortByPType), - Exporter::ExportFormatEntry( "gltf2", "GL Transmission Format v. 2", "gltf", &ExportSceneGLTF2, + Exporter::ExportFormatEntry( "gltf2", "GL Transmission Format v. 2", "gltf2", &ExportSceneGLTF2, aiProcess_JoinIdenticalVertices | aiProcess_Triangulate | aiProcess_SortByPType), #endif From 2056e56bdbe5cf4316c7cc5d48008d1199a8142f Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 30 Sep 2017 10:45:14 +0200 Subject: [PATCH 198/305] Obj: prepare test to reproduce crash on linux. --- test/unit/utObjImportExport.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/unit/utObjImportExport.cpp b/test/unit/utObjImportExport.cpp index d4d4fbf9e..94bc71f67 100644 --- a/test/unit/utObjImportExport.cpp +++ b/test/unit/utObjImportExport.cpp @@ -254,3 +254,18 @@ TEST_F( utObjImportExport, issue809_vertex_color_Test ) { EXPECT_EQ( aiReturn_SUCCESS, exporter.Export( scene, "obj", ASSIMP_TEST_MODELS_DIR "/OBJ/test.obj" ) ); #endif // ASSIMP_BUILD_NO_EXPORT } + +TEST_F( utObjImportExport, issue1453_segfault ) { + static const std::string ObjModel = + "v 0.0 0.0 0.0" + "v 0.0 0.0 1.0" + "v 0.0 1.0 0.0" + "v 0.0 1.0 1.0" + "v 1.0 0.0 0.0" + "v 1.0 0.0 1.0" + "v 1.0 1.0 0.0" + "v 1.0 1.0 1.0"; + + Assimp::Importer myimporter; + const aiScene* myscene = myimporter.ReadFileFromMemory( ObjModel.c_str(), ObjModel.size(), 0 ); +} From c4e91eb33f811a50c38e89dff79f393a38164aea Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 30 Sep 2017 10:47:23 +0200 Subject: [PATCH 199/305] add some asserts. --- code/AssbinLoader.h | 1 - code/FBXConverter.cpp | 2 +- code/FBXProperties.h | 20 +++++++------------- code/SMDLoader.cpp | 6 ++++-- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/code/AssbinLoader.h b/code/AssbinLoader.h index fc28d1667..2eb7a6488 100644 --- a/code/AssbinLoader.h +++ b/code/AssbinLoader.h @@ -71,7 +71,6 @@ class AssbinImporter : public BaseImporter private: bool shortened; bool compressed; -protected: public: virtual bool CanRead( diff --git a/code/FBXConverter.cpp b/code/FBXConverter.cpp index 850028ae8..1a80fa4f5 100644 --- a/code/FBXConverter.cpp +++ b/code/FBXConverter.cpp @@ -1096,7 +1096,7 @@ void Converter::SetupNodeMetadata( const Model& model, aiNode& nd ) DirectPropertyMap unparsedProperties = props.GetUnparsedProperties(); // create metadata on node - std::size_t numStaticMetaData = 2; + const std::size_t numStaticMetaData = 2; aiMetadata* data = aiMetadata::Alloc( static_cast(unparsedProperties.size() + numStaticMetaData) ); nd.mMetaData = data; int index = 0; diff --git a/code/FBXProperties.h b/code/FBXProperties.h index 91b7c8128..09b8aa94c 100644 --- a/code/FBXProperties.h +++ b/code/FBXProperties.h @@ -62,8 +62,7 @@ class Element; P: "ShininessExponent", "double", "Number", "",0.5 @endvebatim */ -class Property -{ +class Property { protected: Property(); @@ -78,15 +77,13 @@ public: }; template -class TypedProperty : public Property -{ +class TypedProperty : public Property { public: explicit TypedProperty(const T& value) - : value(value) - { + : value(value) { + // empty } -public: const T& Value() const { return value; } @@ -97,21 +94,19 @@ private: typedef std::fbx_unordered_map > DirectPropertyMap; -typedef std::fbx_unordered_map PropertyMap; -typedef std::fbx_unordered_map LazyPropertyMap; +typedef std::fbx_unordered_map PropertyMap; +typedef std::fbx_unordered_map LazyPropertyMap; /** * Represents a property table as can be found in the newer FBX files (Properties60, Properties70) */ -class PropertyTable -{ +class PropertyTable { public: // in-memory property table with no source element PropertyTable(); PropertyTable(const Element& element, std::shared_ptr templateProps); ~PropertyTable(); -public: const Property* Get(const std::string& name) const; // PropertyTable's need not be coupled with FBX elements so this can be NULL @@ -132,7 +127,6 @@ private: const Element* const element; }; - // ------------------------------------------------------------------------------------------------ template inline diff --git a/code/SMDLoader.cpp b/code/SMDLoader.cpp index 4502e8d7a..60e3f63c8 100644 --- a/code/SMDLoader.cpp +++ b/code/SMDLoader.cpp @@ -650,12 +650,14 @@ void SMDImporter::ComputeAbsoluteBoneTransformations() // create output materials void SMDImporter::CreateOutputMaterials() { + ai_assert( nullptr != pScene ); + pScene->mNumMaterials = (unsigned int)aszTextures.size(); pScene->mMaterials = new aiMaterial*[std::max(1u, pScene->mNumMaterials)]; - for (unsigned int iMat = 0; iMat < pScene->mNumMaterials;++iMat) - { + for (unsigned int iMat = 0; iMat < pScene->mNumMaterials; ++iMat) { aiMaterial* pcMat = new aiMaterial(); + ai_assert( nullptr != pcMat ); pScene->mMaterials[iMat] = pcMat; aiString szName; From a9e883627116e00c4fb273e79b838f5100dd75d1 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sat, 30 Sep 2017 23:28:02 +1000 Subject: [PATCH 200/305] Added -fPIC flag to C compilers for GCC and clang. Removed -pedantic flag from some compilers. --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b404e6916..a33d1431f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,9 +204,11 @@ ELSEIF(MSVC) # enable multi-core compilation with MSVC add_compile_options(/MP) ELSEIF ( "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" ) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fvisibility=hidden -fPIC -Wall -Wno-long-long -pedantic -std=c++11" ) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fvisibility=hidden -fPIC -Wall -Wno-long-long -std=c++11" ) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") ELSEIF( CMAKE_COMPILER_IS_MINGW ) - SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -Wall -Wno-long-long -pedantic -std=c++11" ) + SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -Wall -Wno-long-long -std=c++11" ) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") ADD_DEFINITIONS( -U__STRICT_ANSI__ ) ENDIF() From 9088deeb1d7aab0de902f003a4acf042dcda9b27 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sat, 30 Sep 2017 23:29:56 +1000 Subject: [PATCH 201/305] Eliminated all warnings under clang with default settings. One remains in the included zlib contrib project. --- code/MMDPmxParser.cpp | 18 +++++++++--------- code/glTFAsset.h | 4 +++- contrib/unzip/unzip.c | 21 +++++++++++++++++++-- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/code/MMDPmxParser.cpp b/code/MMDPmxParser.cpp index c38bdf49a..3af23529a 100644 --- a/code/MMDPmxParser.cpp +++ b/code/MMDPmxParser.cpp @@ -216,8 +216,8 @@ namespace pmx void PmxMaterial::Read(std::istream *stream, PmxSetting *setting) { - this->material_name = std::move(ReadString(stream, setting->encoding)); - this->material_english_name = std::move(ReadString(stream, setting->encoding)); + this->material_name = ReadString(stream, setting->encoding); + this->material_english_name = ReadString(stream, setting->encoding); stream->read((char*) this->diffuse, sizeof(float) * 4); stream->read((char*) this->specular, sizeof(float) * 3); stream->read((char*) &this->specularlity, sizeof(float)); @@ -236,7 +236,7 @@ namespace pmx else { this->toon_texture_index = ReadIndex(stream, setting->texture_index_size); } - this->memo = std::move(ReadString(stream, setting->encoding)); + this->memo = ReadString(stream, setting->encoding); stream->read((char*) &this->index_count, sizeof(int)); } @@ -253,8 +253,8 @@ namespace pmx void PmxBone::Read(std::istream *stream, PmxSetting *setting) { - this->bone_name = std::move(ReadString(stream, setting->encoding)); - this->bone_english_name = std::move(ReadString(stream, setting->encoding)); + this->bone_name = ReadString(stream, setting->encoding); + this->bone_english_name = ReadString(stream, setting->encoding); stream->read((char*) this->position, sizeof(float) * 3); this->parent_index = ReadIndex(stream, setting->bone_index_size); stream->read((char*) &this->level, sizeof(int)); @@ -528,10 +528,10 @@ namespace pmx this->setting.Read(stream); // モデル情報 - this->model_name = std::move(ReadString(stream, setting.encoding)); - this->model_english_name = std::move(ReadString(stream, setting.encoding)); - this->model_comment = std::move(ReadString(stream, setting.encoding)); - this->model_english_comment = std::move(ReadString(stream, setting.encoding)); + this->model_name = ReadString(stream, setting.encoding); + this->model_english_name = ReadString(stream, setting.encoding); + this->model_comment = ReadString(stream, setting.encoding); + this->model_english_comment = ReadString(stream, setting.encoding); // 頂点 stream->read((char*) &vertex_count, sizeof(int)); diff --git a/code/glTFAsset.h b/code/glTFAsset.h index 60107bee6..e018e5ace 100644 --- a/code/glTFAsset.h +++ b/code/glTFAsset.h @@ -232,7 +232,9 @@ namespace glTF case ComponentType_UNSIGNED_BYTE: return 1; default: - throw DeadlyImportError("GLTF: Unsupported Component Type "+t); + std::string err = "GLTF: Unsupported Component Type "; + err += t; + throw DeadlyImportError(err); } } diff --git a/contrib/unzip/unzip.c b/contrib/unzip/unzip.c index e9ad1ff99..085d79a02 100644 --- a/contrib/unzip/unzip.c +++ b/contrib/unzip/unzip.c @@ -608,10 +608,16 @@ local int unzlocal_GetCurrentFileInfoInternal (file, /* we check the magic */ if (err==UNZ_OK) + { if (unzlocal_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + { err=UNZ_ERRNO; + } else if (uMagic!=0x02014b50) + { err=UNZ_BADZIPFILE; + } + } if (unzlocal_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) err=UNZ_ERRNO; @@ -688,18 +694,21 @@ local int unzlocal_GetCurrentFileInfoInternal (file, uSizeRead = extraFieldBufferSize; if (lSeek!=0) + { if (ZSEEK(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) lSeek=0; else err=UNZ_ERRNO; + } if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) if (ZREAD(s->z_filefunc, s->filestream,extraField,uSizeRead)!=uSizeRead) err=UNZ_ERRNO; lSeek += file_info.size_file_extra - uSizeRead; } else + { lSeek+=file_info.size_file_extra; - + } if ((err==UNZ_OK) && (szComment!=NULL)) { @@ -710,20 +719,26 @@ local int unzlocal_GetCurrentFileInfoInternal (file, uSizeRead = file_info.size_file_comment; } else + { uSizeRead = commentBufferSize; + } if (lSeek!=0) + { if (ZSEEK(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) lSeek=0; else err=UNZ_ERRNO; + } if ((file_info.size_file_comment>0) && (commentBufferSize>0)) if (ZREAD(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) err=UNZ_ERRNO; lSeek+=file_info.size_file_comment - uSizeRead; } else + { lSeek+=file_info.size_file_comment; + } if ((err==UNZ_OK) && (pfile_info!=NULL)) *pfile_info=file_info; @@ -977,10 +992,12 @@ local int unzlocal_CheckCurrentFileCoherencyHeader (s,piSizeVar, if (err==UNZ_OK) + { if (unzlocal_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) err=UNZ_ERRNO; else if (uMagic!=0x04034b50) err=UNZ_BADZIPFILE; + } if (unzlocal_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) err=UNZ_ERRNO; @@ -1239,7 +1256,7 @@ extern int ZEXPORT unzReadCurrentFile (file, buf, len) return UNZ_PARAMERROR; - if ((pfile_in_zip_read_info->read_buffer == NULL)) + if (pfile_in_zip_read_info->read_buffer == NULL) return UNZ_END_OF_LIST_OF_FILE; if (len==0) return 0; From 45d93701f898343e804e61e2c03198fba27cd7ff Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 26 Sep 2017 18:33:00 +0300 Subject: [PATCH 202/305] Open3DGC: Fix some uninitialized variable warnings --- contrib/Open3DGC/o3dgcSC3DMCDecoder.inl | 2 +- contrib/Open3DGC/o3dgcSC3DMCEncoder.inl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/Open3DGC/o3dgcSC3DMCDecoder.inl b/contrib/Open3DGC/o3dgcSC3DMCDecoder.inl index d36b62f2b..aa6f24b73 100644 --- a/contrib/Open3DGC/o3dgcSC3DMCDecoder.inl +++ b/contrib/Open3DGC/o3dgcSC3DMCDecoder.inl @@ -425,7 +425,7 @@ namespace o3dgc const AdjacencyInfo & v2T = m_triangleListDecoder.GetVertexToTriangle(); const T * const triangles = ifs.GetCoordIndex(); Vec3 p1, p2, p3, n0, nt; - long na0, nb0; + long na0 = 0, nb0 = 0; Real rna0, rnb0, norm0; char ni0 = 0, ni1 = 0; long a, b, c; diff --git a/contrib/Open3DGC/o3dgcSC3DMCEncoder.inl b/contrib/Open3DGC/o3dgcSC3DMCEncoder.inl index b2c438814..2d30b05a2 100644 --- a/contrib/Open3DGC/o3dgcSC3DMCEncoder.inl +++ b/contrib/Open3DGC/o3dgcSC3DMCEncoder.inl @@ -762,8 +762,8 @@ namespace o3dgc const Real * const originalNormals = ifs.GetNormal(); Vec3 p1, p2, p3, n0, nt; Vec3 n1; - long na0, nb0; - Real rna0, rnb0, na1, nb1, norm0, norm1; + long na0 = 0, nb0 = 0; + Real rna0, rnb0, na1 = 0, nb1 = 0, norm0, norm1; char ni0 = 0, ni1 = 0; long a, b, c, v; m_predictors.Clear(); From 77ce6e562d22d042cd79b60f73bf885e711dcc41 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 30 Sep 2017 16:35:11 +0300 Subject: [PATCH 203/305] Fix CMAKE option name --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b404e6916..8b16c278d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,7 @@ OPTION ( ASSIMP_COVERALLS "Enable this to measure test coverage." OFF ) -OPTION ( ASSIMP_WERRRO +OPTION ( ASSIMP_WERROR "Treat warnings as errors." OFF ) From f7b2380f864ff620ab7e4ddafb24715e5e99c244 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 30 Sep 2017 16:35:41 +0300 Subject: [PATCH 204/305] travis: Treat warnings as errors --- .travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.sh b/.travis.sh index 7161fd28d..2b09da35e 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ERROR=ON } if [ $ANDROID ]; then From 3f4663e369bf64a989fb386301574a8b6ce51cee Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Mon, 2 Oct 2017 09:33:51 +0200 Subject: [PATCH 205/305] closes https://github.com/assimp/assimp/issues/1467. --- code/OpenGEXImporter.cpp | 18 ++++++++++++++---- code/OpenGEXImporter.h | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 93c0169d2..7c4d0f982 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -261,6 +261,7 @@ OpenGEXImporter::OpenGEXImporter() , m_nodeChildMap() , m_meshCache() , m_mesh2refMap() +, m_material2refMap() , m_ctx( nullptr ) , m_metrics() , m_currentNode( nullptr ) @@ -482,7 +483,10 @@ void OpenGEXImporter::handleNameNode( DDLNode *node, aiScene *pScene ) { || m_tokenType == Grammar::CameraNodeToken ) { m_currentNode->mName.Set( name.c_str() ); } else if( m_tokenType == Grammar::MaterialToken ) { - + aiString aiName; + aiName.Set( name ); + m_currentMaterial->AddProperty( &aiName, AI_MATKEY_NAME ); + m_material2refMap[ name ] = m_materialCache.size() - 1; } } } @@ -1050,7 +1054,6 @@ void OpenGEXImporter::handleTextureNode( ODDLParser::DDLNode *node, aiScene *pSc m_currentMaterial->AddProperty( &tex, AI_MATKEY_TEXTURE_DIFFUSE( 0 ) ); } else if( prop->m_value->getString() == Grammar::SpecularPowerTextureToken ) { m_currentMaterial->AddProperty( &tex, AI_MATKEY_TEXTURE_SPECULAR( 0 ) ); - } else if( prop->m_value->getString() == Grammar::EmissionTextureToken ) { m_currentMaterial->AddProperty( &tex, AI_MATKEY_TEXTURE_EMISSIVE( 0 ) ); } else if( prop->m_value->getString() == Grammar::OpacyTextureToken ) { @@ -1166,7 +1169,7 @@ void OpenGEXImporter::resolveReferences() { if( nullptr != currentRefInfo ) { aiNode *node( currentRefInfo->m_node ); if( RefInfo::MeshRef == currentRefInfo->m_type ) { - for( size_t i = 0; i < currentRefInfo->m_Names.size(); i++ ) { + for( size_t i = 0; i < currentRefInfo->m_Names.size(); ++i ) { const std::string &name( currentRefInfo->m_Names[ i ] ); ReferenceMap::const_iterator it( m_mesh2refMap.find( name ) ); if( m_mesh2refMap.end() != it ) { @@ -1175,7 +1178,14 @@ void OpenGEXImporter::resolveReferences() { } } } else if( RefInfo::MaterialRef == currentRefInfo->m_type ) { - // ToDo! + for ( size_t i = 0; i < currentRefInfo->m_Names.size(); ++i ) { + const std::string name( currentRefInfo->m_Names[ i ] ); + ReferenceMap::const_iterator it( m_material2refMap.find( name ) ); + if ( m_material2refMap.end() != it ) { + unsigned int matIdx = static_cast< unsigned int >( m_material2refMap[ name ] ); + m_currentMesh->mMaterialIndex = matIdx; + } + } } else { throw DeadlyImportError( "Unknown reference info to resolve." ); } diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index e3a0735f6..bb6b45140 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -185,6 +185,7 @@ private: std::vector m_meshCache; typedef std::map ReferenceMap; std::map m_mesh2refMap; + std::map m_material2refMap; ODDLParser::Context *m_ctx; MetricInfo m_metrics[ MetricInfo::Max ]; From 6ec25be0a6a21595e878701c776f0ea8c0d3ea8a Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Mon, 2 Oct 2017 09:41:01 +0200 Subject: [PATCH 206/305] OpenGEX: improve logging to be able to detect error-prone situations. --- code/OpenGEXImporter.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 7c4d0f982..d9c58e91c 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -1182,8 +1182,16 @@ void OpenGEXImporter::resolveReferences() { const std::string name( currentRefInfo->m_Names[ i ] ); ReferenceMap::const_iterator it( m_material2refMap.find( name ) ); if ( m_material2refMap.end() != it ) { - unsigned int matIdx = static_cast< unsigned int >( m_material2refMap[ name ] ); - m_currentMesh->mMaterialIndex = matIdx; + if ( nullptr != m_currentMesh ) { + unsigned int matIdx = static_cast< unsigned int >( m_material2refMap[ name ] ); + if ( m_currentMesh->mMaterialIndex != 0 ) { + DefaultLogger::get()->warn( "Override of material reference in current mesh by material reference." ); + } + m_currentMesh->mMaterialIndex = matIdx; + } else { + DefaultLogger::get()->warn( "Cannot resolve material reference, because no current mesh is there." ); + + } } } } else { From 4652b66bb579bf09c921e498f2dd4c21f8669b62 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 12 Sep 2017 18:57:44 +0300 Subject: [PATCH 207/305] Add AddressSanitizer option to CMake --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd5c86140..a4248eb1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,10 @@ OPTION ( ASSIMP_WERROR "Treat warnings as errors." OFF ) +OPTION ( ASSIMP_ASAN + "Enable AddressSanitizer." + OFF +) OPTION ( SYSTEM_IRRXML "Use system installed Irrlicht/IrrXML library." OFF @@ -223,6 +227,11 @@ if (ASSIMP_WERROR) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") endif() +if (ASSIMP_ASAN) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") +endif() + INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) From fff800f9ab8606c0f43cd92de1b291fec1796b1c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 12 Sep 2017 19:09:22 +0300 Subject: [PATCH 208/305] Enable AddressSanitizer for Linux clang build --- .travis.sh | 2 +- .travis.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.sh b/.travis.sh index 2b09da35e..f19c3a000 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ERROR=ON + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ERROR=ON -DASSIMP_ASAN=$ASAN } if [ $ANDROID ]; then diff --git a/.travis.yml b/.travis.yml index cf0988517..d59689f78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,16 +39,16 @@ matrix: include: - os: linux compiler: gcc - env: LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON + env: LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON ASAN=OFF - os: linux compiler: gcc - env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON - os: linux compiler: gcc - env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF - os: linux compiler: gcc - env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From 5ecab20bd00692afff327875d360c94aec6b6016 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Tue, 12 Sep 2017 19:00:44 +0300 Subject: [PATCH 209/305] Fix delete / delete[] mismatch in glTFAsset --- code/glTFAsset.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index 9284ccb02..61afebfb4 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -341,7 +341,7 @@ inline bool Buffer::LoadFromStream(IOStream& stream, size_t length, size_t baseO stream.Seek(baseOffset, aiOrigin_SET); } - mData.reset(new uint8_t[byteLength]); + mData.reset(new uint8_t[byteLength], std::default_delete()); if (stream.Read(mData.get(), byteLength, 1) != 1) { return false; From da96b32fb97e76f593e88e6093992db6500f2f7d Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 09:38:07 +0300 Subject: [PATCH 210/305] Fix out-of-bounds read in MaterialSystem unit test --- test/unit/utMaterialSystem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/utMaterialSystem.cpp b/test/unit/utMaterialSystem.cpp index f6b534f4f..80408db00 100644 --- a/test/unit/utMaterialSystem.cpp +++ b/test/unit/utMaterialSystem.cpp @@ -73,7 +73,7 @@ TEST_F(MaterialSystemTest, testFloatArrayProperty) { float pf[] = {0.0f,1.0f,2.0f,3.0f}; unsigned int pMax = sizeof(pf) / sizeof(float); - this->pcMat->AddProperty(&pf,pMax,"testKey2"); + this->pcMat->AddProperty(pf,pMax,"testKey2"); pf[0] = pf[1] = pf[2] = pf[3] = 12.0f; EXPECT_EQ(AI_SUCCESS, pcMat->Get("testKey2",0,0,pf,&pMax)); @@ -97,7 +97,7 @@ TEST_F(MaterialSystemTest, testIntArrayProperty) { int pf[] = {0,1,2,3}; unsigned int pMax = sizeof(pf) / sizeof(int); - this->pcMat->AddProperty(&pf,pMax,"testKey4"); + this->pcMat->AddProperty(pf,pMax,"testKey4"); pf[0] = pf[1] = pf[2] = pf[3] = 12; EXPECT_EQ(AI_SUCCESS, pcMat->Get("testKey4",0,0,pf,&pMax)); From efd861253d9fc89f64335abbeb08786201156428 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 10:33:49 +0300 Subject: [PATCH 211/305] Fix delete / delete[] mismatches in MakeVerboseFormat --- code/MakeVerboseFormat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/MakeVerboseFormat.cpp b/code/MakeVerboseFormat.cpp index ee82caafe..720d44519 100644 --- a/code/MakeVerboseFormat.cpp +++ b/code/MakeVerboseFormat.cpp @@ -193,14 +193,14 @@ bool MakeVerboseFormatProcess::MakeVerboseFormat(aiMesh* pcMesh) p = 0; while (pcMesh->HasTextureCoords(p)) { - delete pcMesh->mTextureCoords[p]; + delete[] pcMesh->mTextureCoords[p]; pcMesh->mTextureCoords[p] = apvTextureCoords[p]; ++p; } p = 0; while (pcMesh->HasVertexColors(p)) { - delete pcMesh->mColors[p]; + delete[] pcMesh->mColors[p]; pcMesh->mColors[p] = apvColorSets[p]; ++p; } From 1095ec454b9e81d8336fa41ab7109f2226a754f0 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 14 Sep 2017 10:34:16 +0300 Subject: [PATCH 212/305] Fix delete / delete[] mismatches in glTF2 importer --- code/glTF2Asset.inl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 264a32d4d..ac0f353dc 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -357,7 +357,7 @@ inline bool Buffer::LoadFromStream(IOStream& stream, size_t length, size_t baseO stream.Seek(baseOffset, aiOrigin_SET); } - mData.reset(new uint8_t[byteLength]); + mData.reset(new uint8_t[byteLength], std::default_delete()); if (stream.Read(mData.get(), byteLength, 1) != 1) { return false; @@ -451,7 +451,7 @@ inline void Buffer::Grow(size_t amount) if (amount <= 0) return; uint8_t* b = new uint8_t[byteLength + amount]; if (mData) memcpy(b, mData.get(), byteLength); - mData.reset(b); + mData.reset(b, std::default_delete()); byteLength += amount; } From 29e46e4bb819f3499aa363c8df433b8c1529aa11 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 1 Oct 2017 19:25:43 +1100 Subject: [PATCH 213/305] Addressed asan failures caused by misuse of APIs within unit tests. --- test/unit/utObjTools.cpp | 3 ++- test/unit/utSharedPPData.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/utObjTools.cpp b/test/unit/utObjTools.cpp index b359cfa3a..2d9e2779f 100644 --- a/test/unit/utObjTools.cpp +++ b/test/unit/utObjTools.cpp @@ -106,8 +106,9 @@ TEST_F( utObjTools, countComponents_TwoLines_Success ) { TestObjFileParser test_parser; std::string data( "-2.061493116917992e-15 -0.9009688496589661 \\\n-0.4338837265968323" ); std::vector buffer; - buffer.resize( data.size() ); + buffer.resize( data.size() + 1 ); ::memcpy( &buffer[ 0 ], &data[ 0 ], data.size() ); + buffer[ buffer.size() - 1 ] = '\0'; test_parser.setBuffer( buffer ); size_t numComps = test_parser.testGetNumComponentsInDataDefinition(); diff --git a/test/unit/utSharedPPData.cpp b/test/unit/utSharedPPData.cpp index e075365b8..495faa7ac 100644 --- a/test/unit/utSharedPPData.cpp +++ b/test/unit/utSharedPPData.cpp @@ -92,7 +92,7 @@ TEST_F(SharedPPDataTest, testPODProperty) // ------------------------------------------------------------------------------------------------ TEST_F(SharedPPDataTest, testPropertyPointer) { - int *i = new int[35]; + int *i = new int; shared->AddProperty("test16",i); int* o; EXPECT_TRUE(shared->GetProperty("test16",o)); From 1eb7eceddfc0b7f72f2055f4c400bbd24ea7b46f Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 1 Oct 2017 23:16:21 +1100 Subject: [PATCH 214/305] Addressed a number of memory leaks identified in unit tests by asan --- code/B3DImporter.cpp | 20 ++++++++++++++++++++ code/B3DImporter.h | 2 ++ code/PretransformVertices.cpp | 2 +- test/unit/TestModelFactory.h | 8 ++++---- test/unit/utIssues.cpp | 1 + test/unit/utObjImportExport.cpp | 9 +++++++++ test/unit/utRemoveVCProcess.cpp | 3 ++- test/unit/utSceneCombiner.cpp | 7 +++++-- 8 files changed, 44 insertions(+), 8 deletions(-) diff --git a/code/B3DImporter.cpp b/code/B3DImporter.cpp index 5de7bd67c..d15676128 100644 --- a/code/B3DImporter.cpp +++ b/code/B3DImporter.cpp @@ -82,6 +82,20 @@ static const aiImporterDesc desc = { //#define DEBUG_B3D +template +void DeleteAllBarePointers(std::vector& x) +{ + for(auto p : x) + { + delete p; + } +} + +B3DImporter::~B3DImporter() +{ + DeleteAllBarePointers(_animations); +} + // ------------------------------------------------------------------------------------------------ bool B3DImporter::CanRead( const std::string& pFile, IOSystem* /*pIOHandler*/, bool /*checkSig*/) const{ @@ -558,13 +572,19 @@ aiNode *B3DImporter::ReadNODE( aiNode *parent ){ void B3DImporter::ReadBB3D( aiScene *scene ){ _textures.clear(); + _materials.clear(); _vertices.clear(); + _meshes.clear(); + DeleteAllBarePointers(_nodes); _nodes.clear(); + _nodeAnims.clear(); + + DeleteAllBarePointers(_animations); _animations.clear(); string t=ReadChunk(); diff --git a/code/B3DImporter.h b/code/B3DImporter.h index 4d3576dc3..94644edd4 100644 --- a/code/B3DImporter.h +++ b/code/B3DImporter.h @@ -59,6 +59,8 @@ namespace Assimp{ class B3DImporter : public BaseImporter{ public: + B3DImporter() = default; + virtual ~B3DImporter(); virtual bool CanRead( const std::string& pFile, IOSystem* pIOHandler, bool checkSig) const; diff --git a/code/PretransformVertices.cpp b/code/PretransformVertices.cpp index 7bfed4292..5fc294618 100644 --- a/code/PretransformVertices.cpp +++ b/code/PretransformVertices.cpp @@ -634,7 +634,7 @@ void PretransformVertices::Execute( aiScene* pScene) aiNode* newRoot = new aiNode(); newRoot->mName = pScene->mRootNode->mName; delete pScene->mRootNode; - pScene->mRootNode = new aiNode(); + pScene->mRootNode = newRoot; if (1 == pScene->mNumMeshes && !pScene->mNumLights && !pScene->mNumCameras) { diff --git a/test/unit/TestModelFactory.h b/test/unit/TestModelFactory.h index 6c47dcf91..f848f5536 100644 --- a/test/unit/TestModelFactory.h +++ b/test/unit/TestModelFactory.h @@ -60,7 +60,7 @@ public: static aiScene *createDefaultTestModel( float &opacity ) { aiScene *scene( new aiScene ); scene->mNumMaterials = 1; - scene->mMaterials = new aiMaterial*; + scene->mMaterials = new aiMaterial*[scene->mNumMaterials]; scene->mMaterials[ 0 ] = new aiMaterial; aiColor3D color( 1, 0, 0 ); EXPECT_EQ( AI_SUCCESS, scene->mMaterials[ 0 ]->AddProperty( &color, 1, AI_MATKEY_COLOR_DIFFUSE ) ); @@ -70,7 +70,7 @@ public: EXPECT_EQ( AI_SUCCESS, scene->mMaterials[ 0 ]->AddProperty( &opacity, 1, AI_MATKEY_OPACITY ) ); scene->mNumMeshes = 1; - scene->mMeshes = new aiMesh*; + scene->mMeshes = new aiMesh*[scene->mNumMeshes]; scene->mMeshes[ 0 ] = new aiMesh; scene->mMeshes[ 0 ]->mMaterialIndex = 0; scene->mMeshes[ 0 ]->mPrimitiveTypes = aiPrimitiveType_TRIANGLE; @@ -80,7 +80,7 @@ public: scene->mMeshes[ 0 ]->mVertices[ 1 ] = aiVector3D( 0, 1, 0 ); scene->mMeshes[ 0 ]->mVertices[ 2 ] = aiVector3D( 0, 0, 1 ); scene->mMeshes[ 0 ]->mNumFaces = 1; - scene->mMeshes[ 0 ]->mFaces = new aiFace; + scene->mMeshes[ 0 ]->mFaces = new aiFace[scene->mMeshes[ 0 ]->mNumFaces]; scene->mMeshes[ 0 ]->mFaces[ 0 ].mNumIndices = 3; scene->mMeshes[ 0 ]->mFaces[ 0 ].mIndices = new unsigned int[ 3 ]; scene->mMeshes[ 0 ]->mFaces[ 0 ].mIndices[ 0 ] = 0; @@ -89,7 +89,7 @@ public: scene->mRootNode = new aiNode; scene->mRootNode->mNumMeshes = 1; - scene->mRootNode->mMeshes = new unsigned int( 0 ); + scene->mRootNode->mMeshes = new unsigned int[scene->mRootNode->mNumMeshes]{ 0 }; return scene; } diff --git a/test/unit/utIssues.cpp b/test/unit/utIssues.cpp index a3e68fe1a..2feef922b 100644 --- a/test/unit/utIssues.cpp +++ b/test/unit/utIssues.cpp @@ -74,6 +74,7 @@ TEST_F( utIssues, OpacityBugWhenExporting_727 ) { EXPECT_EQ( AI_SUCCESS, newScene->mMaterials[ 0 ]->Get( AI_MATKEY_OPACITY, newOpacity ) ); EXPECT_EQ( opacity, newOpacity ); } + delete scene; } #endif // ASSIMP_BUILD_NO_EXPORT diff --git a/test/unit/utObjImportExport.cpp b/test/unit/utObjImportExport.cpp index d4d4fbf9e..7099252bb 100644 --- a/test/unit/utObjImportExport.cpp +++ b/test/unit/utObjImportExport.cpp @@ -237,6 +237,15 @@ TEST_F( utObjImportExport, obj_import_test ) { differ.showReport(); m_im->FreeScene(); + for(unsigned int i = 0; i < expected->mNumMeshes; ++i) + { + delete expected->mMeshes[i]; + } + delete[] expected->mMeshes; + expected->mMeshes = nullptr; + delete[] expected->mMaterials; + expected->mMaterials = nullptr; + delete expected; } TEST_F( utObjImportExport, issue1111_no_mat_name_Test ) { diff --git a/test/unit/utRemoveVCProcess.cpp b/test/unit/utRemoveVCProcess.cpp index a42c3e858..c78e80d3f 100644 --- a/test/unit/utRemoveVCProcess.cpp +++ b/test/unit/utRemoveVCProcess.cpp @@ -72,4 +72,5 @@ TEST_F( utRevmoveVCProcess, issue1266_ProcessMeshTest_NoCrash ) { scene->mMeshes[ 0 ] = mesh; RemoveVCProcess *process = new RemoveVCProcess; process->Execute( scene ); -} \ No newline at end of file + delete scene; +} diff --git a/test/unit/utSceneCombiner.cpp b/test/unit/utSceneCombiner.cpp index f0e4d5d90..3a283515e 100644 --- a/test/unit/utSceneCombiner.cpp +++ b/test/unit/utSceneCombiner.cpp @@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "UnitTestPCH.h" #include #include +#include using namespace ::Assimp; @@ -63,8 +64,10 @@ TEST_F( utSceneCombiner, MergeMeshes_ValidNames_Test ) { mesh3->mName.Set( "mesh_3" ); merge_list.push_back( mesh3 ); - aiMesh *out( nullptr ); - SceneCombiner::MergeMeshes( &out, 0, merge_list.begin(), merge_list.end() ); + std::unique_ptr out; + aiMesh* ptr = nullptr; + SceneCombiner::MergeMeshes( &ptr, 0, merge_list.begin(), merge_list.end() ); + out.reset(ptr); std::string outName = out->mName.C_Str(); EXPECT_EQ( "mesh_1.mesh_2.mesh_3", outName ); } From 5804667dbb4798c7c20c3ef51e4e1767148f1c7c Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 1 Oct 2017 17:51:13 +1100 Subject: [PATCH 215/305] Addressed some mismatched news/deletes caused by the new glTF2 sources. --- code/glTF2Asset.h | 2 +- code/glTF2Asset.inl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index e2b61c646..63282dc6e 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -511,7 +511,7 @@ namespace glTF2 /// \fn ~SEncodedRegion() /// Destructor. - ~SEncodedRegion() { delete [] DecodedData; } + ~SEncodedRegion() { delete[] DecodedData; } }; /******************* Variables *******************/ diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index ac0f353dc..3082ebfab 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -313,7 +313,7 @@ inline void Buffer::Read(Value& obj, Asset& r) if (dataURI.base64) { uint8_t* data = 0; this->byteLength = Util::DecodeBase64(dataURI.data, dataURI.dataLength, data); - this->mData.reset(data); + this->mData.reset(data, std::default_delete()); if (statedLength > 0 && this->byteLength != statedLength) { throw DeadlyImportError("GLTF: buffer \"" + id + "\", expected " + to_string(statedLength) + @@ -326,7 +326,7 @@ inline void Buffer::Read(Value& obj, Asset& r) " bytes, but found " + to_string(dataURI.dataLength)); } - this->mData.reset(new uint8_t[dataURI.dataLength]); + this->mData.reset(new uint8_t[dataURI.dataLength], std::default_delete()); memcpy( this->mData.get(), dataURI.data, dataURI.dataLength ); } } @@ -432,7 +432,7 @@ uint8_t* new_data; // Copy data which place after replacing part. memcpy(&new_data[pBufferData_Offset + pReplace_Count], &mData.get()[pBufferData_Offset + pBufferData_Count], pBufferData_Offset); // Apply new data - mData.reset(new_data); + mData.reset(new_data, std::default_delete()); byteLength = new_data_size; return true; From 799f0a3ac8e3d0161402e4ba3e112ea41da95c9c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 2 Oct 2017 11:40:57 +0300 Subject: [PATCH 216/305] Fix warnings-as-errors flag on MSVC --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd5c86140..ba219a443 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -219,8 +219,12 @@ if (ASSIMP_COVERALLS) endif() if (ASSIMP_WERROR) + IF (MSVC) + add_compile_options(/WX) + ELSE() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + ENDIF() endif() INCLUDE (FindPkgMacros) From 003c728dafd69430b4f3c7bbffa669a8c2be4764 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 2 Oct 2017 11:43:29 +0300 Subject: [PATCH 217/305] appveyor: Treat warnings as errors --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 991cf5bc1..b8828710b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,8 +29,8 @@ install: build_script: - cd c:\projects\assimp -- if "%platform%" equ "x64" (cmake CMakeLists.txt -G "Visual Studio %Configuration% Win64") -- if "%platform%" equ "x86" (cmake CMakeLists.txt -G "Visual Studio %Configuration%") +- if "%platform%" equ "x64" (cmake CMakeLists.txt -DASSIMP_WERROR=ON -G "Visual Studio %Configuration% Win64") +- if "%platform%" equ "x86" (cmake CMakeLists.txt -DASSIMP_WERROR=ON -G "Visual Studio %Configuration%") - if "%platform%" equ "x64" (msbuild /m /p:Configuration=Release /p:Platform="x64" Assimp.sln) - if "%platform%" equ "x86" (msbuild /m /p:Configuration=Release /p:Platform="Win32" Assimp.sln) From b5db7d3649f06abd153a2f81fd4da4f4f70da3a2 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Mon, 2 Oct 2017 13:08:20 +0300 Subject: [PATCH 218/305] Disable warning 4351 on MSVC 2013 --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba219a443..771800205 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,6 +203,11 @@ IF ((CMAKE_C_COMPILER_ID MATCHES "GNU") AND NOT CMAKE_COMPILER_IS_MINGW) ELSEIF(MSVC) # enable multi-core compilation with MSVC add_compile_options(/MP) + + # disable "elements of array '' will be default initialized" warning on MSVC2013 + IF(MSVC12) + add_compile_options(/wd4351) + ENDIF() ELSEIF ( "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" ) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fvisibility=hidden -fPIC -Wall -Wno-long-long -std=c++11" ) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") From bab6ca20856b1fab87680cb19103ccc3bc3527f7 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Wed, 4 Oct 2017 20:33:59 +1100 Subject: [PATCH 219/305] Upgraded zlib to 1.2.11.1 from the develop branch. --- contrib/zlib/.gitignore | 26 + contrib/zlib/CMakeLists.txt | 11 +- contrib/zlib/ChangeLog | 1518 +++++++++++++++++++++++ contrib/zlib/FAQ | 368 ++++++ contrib/zlib/INDEX | 68 + contrib/zlib/Makefile.in | 410 ++++++ contrib/zlib/README | 6 +- contrib/zlib/adler32.c | 21 +- contrib/zlib/compress.c | 42 +- contrib/zlib/configure | 921 ++++++++++++++ contrib/zlib/crc32.c | 47 +- contrib/zlib/deflate.c | 877 ++++++++----- contrib/zlib/deflate.h | 35 +- contrib/zlib/gzguts.h | 23 +- contrib/zlib/gzlib.c | 31 +- contrib/zlib/gzread.c | 160 ++- contrib/zlib/gzwrite.c | 343 +++-- contrib/zlib/infback.c | 4 +- contrib/zlib/inffast.c | 111 +- contrib/zlib/inflate.c | 125 +- contrib/zlib/inflate.h | 11 +- contrib/zlib/inftrees.c | 26 +- contrib/zlib/make_vms.com | 867 +++++++++++++ contrib/zlib/treebuild.xml | 116 ++ contrib/zlib/trees.c | 99 +- contrib/zlib/uncompr.c | 114 +- contrib/zlib/win32/Makefile.gcc | 2 +- contrib/zlib/win32/Makefile.msc | 2 +- contrib/zlib/win32/README-WIN32.txt | 6 +- contrib/zlib/win32/VisualC.txt | 2 +- contrib/zlib/win32/zlib.def | 8 + contrib/zlib/win32/zlib1.rc | 2 +- contrib/zlib/zconf.h.cmakein | 41 +- contrib/zlib/{zconf.in.h => zconf.h.in} | 326 ++++- contrib/zlib/zconf.h.included | 43 +- contrib/zlib/zlib.3 | 149 +++ contrib/zlib/zlib.3.pdf | Bin 0 -> 19320 bytes contrib/zlib/zlib.h | 445 ++++--- contrib/zlib/zlib.map | 94 ++ contrib/zlib/zlib.pc.in | 13 + contrib/zlib/zlib2ansi | 152 +++ contrib/zlib/zutil.c | 53 +- contrib/zlib/zutil.h | 60 +- 43 files changed, 6670 insertions(+), 1108 deletions(-) create mode 100644 contrib/zlib/.gitignore create mode 100644 contrib/zlib/ChangeLog create mode 100644 contrib/zlib/FAQ create mode 100644 contrib/zlib/INDEX create mode 100644 contrib/zlib/Makefile.in create mode 100644 contrib/zlib/configure create mode 100644 contrib/zlib/make_vms.com create mode 100644 contrib/zlib/treebuild.xml rename contrib/zlib/{zconf.in.h => zconf.h.in} (50%) create mode 100644 contrib/zlib/zlib.3 create mode 100644 contrib/zlib/zlib.3.pdf create mode 100644 contrib/zlib/zlib.map create mode 100644 contrib/zlib/zlib.pc.in create mode 100644 contrib/zlib/zlib2ansi diff --git a/contrib/zlib/.gitignore b/contrib/zlib/.gitignore new file mode 100644 index 000000000..b1c7422fe --- /dev/null +++ b/contrib/zlib/.gitignore @@ -0,0 +1,26 @@ +*.diff +*.patch +*.orig +*.rej + +*~ +*.a +*.lo +*.o +*.dylib + +*.gcda +*.gcno +*.gcov + +/example +/example64 +/examplesh +/libz.so* +/minigzip +/minigzip64 +/minigzipsh +/zlib.pc +/configure.log + +.DS_Store diff --git a/contrib/zlib/CMakeLists.txt b/contrib/zlib/CMakeLists.txt index 022d3dfba..5f1368adb 100644 --- a/contrib/zlib/CMakeLists.txt +++ b/contrib/zlib/CMakeLists.txt @@ -10,10 +10,10 @@ endif() project(zlib C) cmake_policy(POP) -set(VERSION "1.2.8") +set(VERSION "1.2.11.1") -option(ASM686 "Enable building i686 assembly implementation for zlib") -option(AMD64 "Enable building amd64 assembly implementation for zlib") +option(ASM686 "Enable building i686 assembly implementation") +option(AMD64 "Enable building amd64 assembly implementation") #set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation directory for executables") #set(INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Installation directory for libraries") @@ -90,7 +90,6 @@ configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/zlib.pc.cmakein ${ZLIB_PC} @ONLY) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h.cmakein ${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY) - include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_SOURCE_DIR}) @@ -191,7 +190,11 @@ if(MINGW) set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) endif(MINGW) +add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) +set_target_properties(zlib PROPERTIES SOVERSION 1) + INSTALL( TARGETS zlibstatic LIBRARY DESTINATION ${ASSIMP_LIB_INSTALL_DIR} ARCHIVE DESTINATION ${ASSIMP_LIB_INSTALL_DIR} diff --git a/contrib/zlib/ChangeLog b/contrib/zlib/ChangeLog new file mode 100644 index 000000000..9c6d95b8d --- /dev/null +++ b/contrib/zlib/ChangeLog @@ -0,0 +1,1518 @@ + + ChangeLog file for zlib + +Changes in 1.2.11.1 (xx Jan 2017) +- + +Changes in 1.2.11 (15 Jan 2017) +- Fix deflate stored bug when pulling last block from window +- Permit immediate deflateParams changes before any deflate input + +Changes in 1.2.10 (2 Jan 2017) +- Avoid warnings on snprintf() return value +- Fix bug in deflate_stored() for zero-length input +- Fix bug in gzwrite.c that produced corrupt gzip files +- Remove files to be installed before copying them in Makefile.in +- Add warnings when compiling with assembler code + +Changes in 1.2.9 (31 Dec 2016) +- Fix contrib/minizip to permit unzipping with desktop API [Zouzou] +- Improve contrib/blast to return unused bytes +- Assure that gzoffset() is correct when appending +- Improve compress() and uncompress() to support large lengths +- Fix bug in test/example.c where error code not saved +- Remedy Coverity warning [Randers-Pehrson] +- Improve speed of gzprintf() in transparent mode +- Fix inflateInit2() bug when windowBits is 16 or 32 +- Change DEBUG macro to ZLIB_DEBUG +- Avoid uninitialized access by gzclose_w() +- Allow building zlib outside of the source directory +- Fix bug that accepted invalid zlib header when windowBits is zero +- Fix gzseek() problem on MinGW due to buggy _lseeki64 there +- Loop on write() calls in gzwrite.c in case of non-blocking I/O +- Add --warn (-w) option to ./configure for more compiler warnings +- Reject a window size of 256 bytes if not using the zlib wrapper +- Fix bug when level 0 used with Z_HUFFMAN or Z_RLE +- Add --debug (-d) option to ./configure to define ZLIB_DEBUG +- Fix bugs in creating a very large gzip header +- Add uncompress2() function, which returns the input size used +- Assure that deflateParams() will not switch functions mid-block +- Dramatically speed up deflation for level 0 (storing) +- Add gzfread(), duplicating the interface of fread() +- Add gzfwrite(), duplicating the interface of fwrite() +- Add deflateGetDictionary() function +- Use snprintf() for later versions of Microsoft C +- Fix *Init macros to use z_ prefix when requested +- Replace as400 with os400 for OS/400 support [Monnerat] +- Add crc32_z() and adler32_z() functions with size_t lengths +- Update Visual Studio project files [AraHaan] + +Changes in 1.2.8 (28 Apr 2013) +- Update contrib/minizip/iowin32.c for Windows RT [Vollant] +- Do not force Z_CONST for C++ +- Clean up contrib/vstudio [Roß] +- Correct spelling error in zlib.h +- Fix mixed line endings in contrib/vstudio + +Changes in 1.2.7.3 (13 Apr 2013) +- Fix version numbers and DLL names in contrib/vstudio/*/zlib.rc + +Changes in 1.2.7.2 (13 Apr 2013) +- Change check for a four-byte type back to hexadecimal +- Fix typo in win32/Makefile.msc +- Add casts in gzwrite.c for pointer differences + +Changes in 1.2.7.1 (24 Mar 2013) +- Replace use of unsafe string functions with snprintf if available +- Avoid including stddef.h on Windows for Z_SOLO compile [Niessink] +- Fix gzgetc undefine when Z_PREFIX set [Turk] +- Eliminate use of mktemp in Makefile (not always available) +- Fix bug in 'F' mode for gzopen() +- Add inflateGetDictionary() function +- Correct comment in deflate.h +- Use _snprintf for snprintf in Microsoft C +- On Darwin, only use /usr/bin/libtool if libtool is not Apple +- Delete "--version" file if created by "ar --version" [Richard G.] +- Fix configure check for veracity of compiler error return codes +- Fix CMake compilation of static lib for MSVC2010 x64 +- Remove unused variable in infback9.c +- Fix argument checks in gzlog_compress() and gzlog_write() +- Clean up the usage of z_const and respect const usage within zlib +- Clean up examples/gzlog.[ch] comparisons of different types +- Avoid shift equal to bits in type (caused endless loop) +- Fix uninitialized value bug in gzputc() introduced by const patches +- Fix memory allocation error in examples/zran.c [Nor] +- Fix bug where gzopen(), gzclose() would write an empty file +- Fix bug in gzclose() when gzwrite() runs out of memory +- Check for input buffer malloc failure in examples/gzappend.c +- Add note to contrib/blast to use binary mode in stdio +- Fix comparisons of differently signed integers in contrib/blast +- Check for invalid code length codes in contrib/puff +- Fix serious but very rare decompression bug in inftrees.c +- Update inflateBack() comments, since inflate() can be faster +- Use underscored I/O function names for WINAPI_FAMILY +- Add _tr_flush_bits to the external symbols prefixed by --zprefix +- Add contrib/vstudio/vc10 pre-build step for static only +- Quote --version-script argument in CMakeLists.txt +- Don't specify --version-script on Apple platforms in CMakeLists.txt +- Fix casting error in contrib/testzlib/testzlib.c +- Fix types in contrib/minizip to match result of get_crc_table() +- Simplify contrib/vstudio/vc10 with 'd' suffix +- Add TOP support to win32/Makefile.msc +- Suport i686 and amd64 assembler builds in CMakeLists.txt +- Fix typos in the use of _LARGEFILE64_SOURCE in zconf.h +- Add vc11 and vc12 build files to contrib/vstudio +- Add gzvprintf() as an undocumented function in zlib +- Fix configure for Sun shell +- Remove runtime check in configure for four-byte integer type +- Add casts and consts to ease user conversion to C++ +- Add man pages for minizip and miniunzip +- In Makefile uninstall, don't rm if preceding cd fails +- Do not return Z_BUF_ERROR if deflateParam() has nothing to write + +Changes in 1.2.7 (2 May 2012) +- Replace use of memmove() with a simple copy for portability +- Test for existence of strerror +- Restore gzgetc_ for backward compatibility with 1.2.6 +- Fix build with non-GNU make on Solaris +- Require gcc 4.0 or later on Mac OS X to use the hidden attribute +- Include unistd.h for Watcom C +- Use __WATCOMC__ instead of __WATCOM__ +- Do not use the visibility attribute if NO_VIZ defined +- Improve the detection of no hidden visibility attribute +- Avoid using __int64 for gcc or solo compilation +- Cast to char * in gzprintf to avoid warnings [Zinser] +- Fix make_vms.com for VAX [Zinser] +- Don't use library or built-in byte swaps +- Simplify test and use of gcc hidden attribute +- Fix bug in gzclose_w() when gzwrite() fails to allocate memory +- Add "x" (O_EXCL) and "e" (O_CLOEXEC) modes support to gzopen() +- Fix bug in test/minigzip.c for configure --solo +- Fix contrib/vstudio project link errors [Mohanathas] +- Add ability to choose the builder in make_vms.com [Schweda] +- Add DESTDIR support to mingw32 win32/Makefile.gcc +- Fix comments in win32/Makefile.gcc for proper usage +- Allow overriding the default install locations for cmake +- Generate and install the pkg-config file with cmake +- Build both a static and a shared version of zlib with cmake +- Include version symbols for cmake builds +- If using cmake with MSVC, add the source directory to the includes +- Remove unneeded EXTRA_CFLAGS from win32/Makefile.gcc [Truta] +- Move obsolete emx makefile to old [Truta] +- Allow the use of -Wundef when compiling or using zlib +- Avoid the use of the -u option with mktemp +- Improve inflate() documentation on the use of Z_FINISH +- Recognize clang as gcc +- Add gzopen_w() in Windows for wide character path names +- Rename zconf.h in CMakeLists.txt to move it out of the way +- Add source directory in CMakeLists.txt for building examples +- Look in build directory for zlib.pc in CMakeLists.txt +- Remove gzflags from zlibvc.def in vc9 and vc10 +- Fix contrib/minizip compilation in the MinGW environment +- Update ./configure for Solaris, support --64 [Mooney] +- Remove -R. from Solaris shared build (possible security issue) +- Avoid race condition for parallel make (-j) running example +- Fix type mismatch between get_crc_table() and crc_table +- Fix parsing of version with "-" in CMakeLists.txt [Snider, Ziegler] +- Fix the path to zlib.map in CMakeLists.txt +- Force the native libtool in Mac OS X to avoid GNU libtool [Beebe] +- Add instructions to win32/Makefile.gcc for shared install [Torri] + +Changes in 1.2.6.1 (12 Feb 2012) +- Avoid the use of the Objective-C reserved name "id" +- Include io.h in gzguts.h for Microsoft compilers +- Fix problem with ./configure --prefix and gzgetc macro +- Include gz_header definition when compiling zlib solo +- Put gzflags() functionality back in zutil.c +- Avoid library header include in crc32.c for Z_SOLO +- Use name in GCC_CLASSIC as C compiler for coverage testing, if set +- Minor cleanup in contrib/minizip/zip.c [Vollant] +- Update make_vms.com [Zinser] +- Remove unnecessary gzgetc_ function +- Use optimized byte swap operations for Microsoft and GNU [Snyder] +- Fix minor typo in zlib.h comments [Rzesniowiecki] + +Changes in 1.2.6 (29 Jan 2012) +- Update the Pascal interface in contrib/pascal +- Fix function numbers for gzgetc_ in zlibvc.def files +- Fix configure.ac for contrib/minizip [Schiffer] +- Fix large-entry detection in minizip on 64-bit systems [Schiffer] +- Have ./configure use the compiler return code for error indication +- Fix CMakeLists.txt for cross compilation [McClure] +- Fix contrib/minizip/zip.c for 64-bit architectures [Dalsnes] +- Fix compilation of contrib/minizip on FreeBSD [Marquez] +- Correct suggested usages in win32/Makefile.msc [Shachar, Horvath] +- Include io.h for Turbo C / Borland C on all platforms [Truta] +- Make version explicit in contrib/minizip/configure.ac [Bosmans] +- Avoid warning for no encryption in contrib/minizip/zip.c [Vollant] +- Minor cleanup up contrib/minizip/unzip.c [Vollant] +- Fix bug when compiling minizip with C++ [Vollant] +- Protect for long name and extra fields in contrib/minizip [Vollant] +- Avoid some warnings in contrib/minizip [Vollant] +- Add -I../.. -L../.. to CFLAGS for minizip and miniunzip +- Add missing libs to minizip linker command +- Add support for VPATH builds in contrib/minizip +- Add an --enable-demos option to contrib/minizip/configure +- Add the generation of configure.log by ./configure +- Exit when required parameters not provided to win32/Makefile.gcc +- Have gzputc return the character written instead of the argument +- Use the -m option on ldconfig for BSD systems [Tobias] +- Correct in zlib.map when deflateResetKeep was added + +Changes in 1.2.5.3 (15 Jan 2012) +- Restore gzgetc function for binary compatibility +- Do not use _lseeki64 under Borland C++ [Truta] +- Update win32/Makefile.msc to build test/*.c [Truta] +- Remove old/visualc6 given CMakefile and other alternatives +- Update AS400 build files and documentation [Monnerat] +- Update win32/Makefile.gcc to build test/*.c [Truta] +- Permit stronger flushes after Z_BLOCK flushes +- Avoid extraneous empty blocks when doing empty flushes +- Permit Z_NULL arguments to deflatePending +- Allow deflatePrime() to insert bits in the middle of a stream +- Remove second empty static block for Z_PARTIAL_FLUSH +- Write out all of the available bits when using Z_BLOCK +- Insert the first two strings in the hash table after a flush + +Changes in 1.2.5.2 (17 Dec 2011) +- fix ld error: unable to find version dependency 'ZLIB_1.2.5' +- use relative symlinks for shared libs +- Avoid searching past window for Z_RLE strategy +- Assure that high-water mark initialization is always applied in deflate +- Add assertions to fill_window() in deflate.c to match comments +- Update python link in README +- Correct spelling error in gzread.c +- Fix bug in gzgets() for a concatenated empty gzip stream +- Correct error in comment for gz_make() +- Change gzread() and related to ignore junk after gzip streams +- Allow gzread() and related to continue after gzclearerr() +- Allow gzrewind() and gzseek() after a premature end-of-file +- Simplify gzseek() now that raw after gzip is ignored +- Change gzgetc() to a macro for speed (~40% speedup in testing) +- Fix gzclose() to return the actual error last encountered +- Always add large file support for windows +- Include zconf.h for windows large file support +- Include zconf.h.cmakein for windows large file support +- Update zconf.h.cmakein on make distclean +- Merge vestigial vsnprintf determination from zutil.h to gzguts.h +- Clarify how gzopen() appends in zlib.h comments +- Correct documentation of gzdirect() since junk at end now ignored +- Add a transparent write mode to gzopen() when 'T' is in the mode +- Update python link in zlib man page +- Get inffixed.h and MAKEFIXED result to match +- Add a ./config --solo option to make zlib subset with no library use +- Add undocumented inflateResetKeep() function for CAB file decoding +- Add --cover option to ./configure for gcc coverage testing +- Add #define ZLIB_CONST option to use const in the z_stream interface +- Add comment to gzdopen() in zlib.h to use dup() when using fileno() +- Note behavior of uncompress() to provide as much data as it can +- Add files in contrib/minizip to aid in building libminizip +- Split off AR options in Makefile.in and configure +- Change ON macro to Z_ARG to avoid application conflicts +- Facilitate compilation with Borland C++ for pragmas and vsnprintf +- Include io.h for Turbo C / Borland C++ +- Move example.c and minigzip.c to test/ +- Simplify incomplete code table filling in inflate_table() +- Remove code from inflate.c and infback.c that is impossible to execute +- Test the inflate code with full coverage +- Allow deflateSetDictionary, inflateSetDictionary at any time (in raw) +- Add deflateResetKeep and fix inflateResetKeep to retain dictionary +- Fix gzwrite.c to accommodate reduced memory zlib compilation +- Have inflate() with Z_FINISH avoid the allocation of a window +- Do not set strm->adler when doing raw inflate +- Fix gzeof() to behave just like feof() when read is not past end of file +- Fix bug in gzread.c when end-of-file is reached +- Avoid use of Z_BUF_ERROR in gz* functions except for premature EOF +- Document gzread() capability to read concurrently written files +- Remove hard-coding of resource compiler in CMakeLists.txt [Blammo] + +Changes in 1.2.5.1 (10 Sep 2011) +- Update FAQ entry on shared builds (#13) +- Avoid symbolic argument to chmod in Makefile.in +- Fix bug and add consts in contrib/puff [Oberhumer] +- Update contrib/puff/zeros.raw test file to have all block types +- Add full coverage test for puff in contrib/puff/Makefile +- Fix static-only-build install in Makefile.in +- Fix bug in unzGetCurrentFileInfo() in contrib/minizip [Kuno] +- Add libz.a dependency to shared in Makefile.in for parallel builds +- Spell out "number" (instead of "nb") in zlib.h for total_in, total_out +- Replace $(...) with `...` in configure for non-bash sh [Bowler] +- Add darwin* to Darwin* and solaris* to SunOS\ 5* in configure [Groffen] +- Add solaris* to Linux* in configure to allow gcc use [Groffen] +- Add *bsd* to Linux* case in configure [Bar-Lev] +- Add inffast.obj to dependencies in win32/Makefile.msc +- Correct spelling error in deflate.h [Kohler] +- Change libzdll.a again to libz.dll.a (!) in win32/Makefile.gcc +- Add test to configure for GNU C looking for gcc in output of $cc -v +- Add zlib.pc generation to win32/Makefile.gcc [Weigelt] +- Fix bug in zlib.h for _FILE_OFFSET_BITS set and _LARGEFILE64_SOURCE not +- Add comment in zlib.h that adler32_combine with len2 < 0 makes no sense +- Make NO_DIVIDE option in adler32.c much faster (thanks to John Reiser) +- Make stronger test in zconf.h to include unistd.h for LFS +- Apply Darwin patches for 64-bit file offsets to contrib/minizip [Slack] +- Fix zlib.h LFS support when Z_PREFIX used +- Add updated as400 support (removed from old) [Monnerat] +- Avoid deflate sensitivity to volatile input data +- Avoid division in adler32_combine for NO_DIVIDE +- Clarify the use of Z_FINISH with deflateBound() amount of space +- Set binary for output file in puff.c +- Use u4 type for crc_table to avoid conversion warnings +- Apply casts in zlib.h to avoid conversion warnings +- Add OF to prototypes for adler32_combine_ and crc32_combine_ [Miller] +- Improve inflateSync() documentation to note indeterminancy +- Add deflatePending() function to return the amount of pending output +- Correct the spelling of "specification" in FAQ [Randers-Pehrson] +- Add a check in configure for stdarg.h, use for gzprintf() +- Check that pointers fit in ints when gzprint() compiled old style +- Add dummy name before $(SHAREDLIBV) in Makefile [Bar-Lev, Bowler] +- Delete line in configure that adds -L. libz.a to LDFLAGS [Weigelt] +- Add debug records in assmebler code [Londer] +- Update RFC references to use http://tools.ietf.org/html/... [Li] +- Add --archs option, use of libtool to configure for Mac OS X [Borstel] + +Changes in 1.2.5 (19 Apr 2010) +- Disable visibility attribute in win32/Makefile.gcc [Bar-Lev] +- Default to libdir as sharedlibdir in configure [Nieder] +- Update copyright dates on modified source files +- Update trees.c to be able to generate modified trees.h +- Exit configure for MinGW, suggesting win32/Makefile.gcc +- Check for NULL path in gz_open [Homurlu] + +Changes in 1.2.4.5 (18 Apr 2010) +- Set sharedlibdir in configure [Torok] +- Set LDFLAGS in Makefile.in [Bar-Lev] +- Avoid mkdir objs race condition in Makefile.in [Bowler] +- Add ZLIB_INTERNAL in front of internal inter-module functions and arrays +- Define ZLIB_INTERNAL to hide internal functions and arrays for GNU C +- Don't use hidden attribute when it is a warning generator (e.g. Solaris) + +Changes in 1.2.4.4 (18 Apr 2010) +- Fix CROSS_PREFIX executable testing, CHOST extract, mingw* [Torok] +- Undefine _LARGEFILE64_SOURCE in zconf.h if it is zero, but not if empty +- Try to use bash or ksh regardless of functionality of /bin/sh +- Fix configure incompatibility with NetBSD sh +- Remove attempt to run under bash or ksh since have better NetBSD fix +- Fix win32/Makefile.gcc for MinGW [Bar-Lev] +- Add diagnostic messages when using CROSS_PREFIX in configure +- Added --sharedlibdir option to configure [Weigelt] +- Use hidden visibility attribute when available [Frysinger] + +Changes in 1.2.4.3 (10 Apr 2010) +- Only use CROSS_PREFIX in configure for ar and ranlib if they exist +- Use CROSS_PREFIX for nm [Bar-Lev] +- Assume _LARGEFILE64_SOURCE defined is equivalent to true +- Avoid use of undefined symbols in #if with && and || +- Make *64 prototypes in gzguts.h consistent with functions +- Add -shared load option for MinGW in configure [Bowler] +- Move z_off64_t to public interface, use instead of off64_t +- Remove ! from shell test in configure (not portable to Solaris) +- Change +0 macro tests to -0 for possibly increased portability + +Changes in 1.2.4.2 (9 Apr 2010) +- Add consistent carriage returns to readme.txt's in masmx86 and masmx64 +- Really provide prototypes for *64 functions when building without LFS +- Only define unlink() in minigzip.c if unistd.h not included +- Update README to point to contrib/vstudio project files +- Move projects/vc6 to old/ and remove projects/ +- Include stdlib.h in minigzip.c for setmode() definition under WinCE +- Clean up assembler builds in win32/Makefile.msc [Rowe] +- Include sys/types.h for Microsoft for off_t definition +- Fix memory leak on error in gz_open() +- Symbolize nm as $NM in configure [Weigelt] +- Use TEST_LDSHARED instead of LDSHARED to link test programs [Weigelt] +- Add +0 to _FILE_OFFSET_BITS and _LFS64_LARGEFILE in case not defined +- Fix bug in gzeof() to take into account unused input data +- Avoid initialization of structures with variables in puff.c +- Updated win32/README-WIN32.txt [Rowe] + +Changes in 1.2.4.1 (28 Mar 2010) +- Remove the use of [a-z] constructs for sed in configure [gentoo 310225] +- Remove $(SHAREDLIB) from LIBS in Makefile.in [Creech] +- Restore "for debugging" comment on sprintf() in gzlib.c +- Remove fdopen for MVS from gzguts.h +- Put new README-WIN32.txt in win32 [Rowe] +- Add check for shell to configure and invoke another shell if needed +- Fix big fat stinking bug in gzseek() on uncompressed files +- Remove vestigial F_OPEN64 define in zutil.h +- Set and check the value of _LARGEFILE_SOURCE and _LARGEFILE64_SOURCE +- Avoid errors on non-LFS systems when applications define LFS macros +- Set EXE to ".exe" in configure for MINGW [Kahle] +- Match crc32() in crc32.c exactly to the prototype in zlib.h [Sherrill] +- Add prefix for cross-compilation in win32/makefile.gcc [Bar-Lev] +- Add DLL install in win32/makefile.gcc [Bar-Lev] +- Allow Linux* or linux* from uname in configure [Bar-Lev] +- Allow ldconfig to be redefined in configure and Makefile.in [Bar-Lev] +- Add cross-compilation prefixes to configure [Bar-Lev] +- Match type exactly in gz_load() invocation in gzread.c +- Match type exactly of zcalloc() in zutil.c to zlib.h alloc_func +- Provide prototypes for *64 functions when building zlib without LFS +- Don't use -lc when linking shared library on MinGW +- Remove errno.h check in configure and vestigial errno code in zutil.h + +Changes in 1.2.4 (14 Mar 2010) +- Fix VER3 extraction in configure for no fourth subversion +- Update zlib.3, add docs to Makefile.in to make .pdf out of it +- Add zlib.3.pdf to distribution +- Don't set error code in gzerror() if passed pointer is NULL +- Apply destination directory fixes to CMakeLists.txt [Lowman] +- Move #cmakedefine's to a new zconf.in.cmakein +- Restore zconf.h for builds that don't use configure or cmake +- Add distclean to dummy Makefile for convenience +- Update and improve INDEX, README, and FAQ +- Update CMakeLists.txt for the return of zconf.h [Lowman] +- Update contrib/vstudio/vc9 and vc10 [Vollant] +- Change libz.dll.a back to libzdll.a in win32/Makefile.gcc +- Apply license and readme changes to contrib/asm686 [Raiter] +- Check file name lengths and add -c option in minigzip.c [Li] +- Update contrib/amd64 and contrib/masmx86/ [Vollant] +- Avoid use of "eof" parameter in trees.c to not shadow library variable +- Update make_vms.com for removal of zlibdefs.h [Zinser] +- Update assembler code and vstudio projects in contrib [Vollant] +- Remove outdated assembler code contrib/masm686 and contrib/asm586 +- Remove old vc7 and vc8 from contrib/vstudio +- Update win32/Makefile.msc, add ZLIB_VER_SUBREVISION [Rowe] +- Fix memory leaks in gzclose_r() and gzclose_w(), file leak in gz_open() +- Add contrib/gcc_gvmat64 for longest_match and inflate_fast [Vollant] +- Remove *64 functions from win32/zlib.def (they're not 64-bit yet) +- Fix bug in void-returning vsprintf() case in gzwrite.c +- Fix name change from inflate.h in contrib/inflate86/inffas86.c +- Check if temporary file exists before removing in make_vms.com [Zinser] +- Fix make install and uninstall for --static option +- Fix usage of _MSC_VER in gzguts.h and zutil.h [Truta] +- Update readme.txt in contrib/masmx64 and masmx86 to assemble + +Changes in 1.2.3.9 (21 Feb 2010) +- Expunge gzio.c +- Move as400 build information to old +- Fix updates in contrib/minizip and contrib/vstudio +- Add const to vsnprintf test in configure to avoid warnings [Weigelt] +- Delete zconf.h (made by configure) [Weigelt] +- Change zconf.in.h to zconf.h.in per convention [Weigelt] +- Check for NULL buf in gzgets() +- Return empty string for gzgets() with len == 1 (like fgets()) +- Fix description of gzgets() in zlib.h for end-of-file, NULL return +- Update minizip to 1.1 [Vollant] +- Avoid MSVC loss of data warnings in gzread.c, gzwrite.c +- Note in zlib.h that gzerror() should be used to distinguish from EOF +- Remove use of snprintf() from gzlib.c +- Fix bug in gzseek() +- Update contrib/vstudio, adding vc9 and vc10 [Kuno, Vollant] +- Fix zconf.h generation in CMakeLists.txt [Lowman] +- Improve comments in zconf.h where modified by configure + +Changes in 1.2.3.8 (13 Feb 2010) +- Clean up text files (tabs, trailing whitespace, etc.) [Oberhumer] +- Use z_off64_t in gz_zero() and gz_skip() to match state->skip +- Avoid comparison problem when sizeof(int) == sizeof(z_off64_t) +- Revert to Makefile.in from 1.2.3.6 (live with the clutter) +- Fix missing error return in gzflush(), add zlib.h note +- Add *64 functions to zlib.map [Levin] +- Fix signed/unsigned comparison in gz_comp() +- Use SFLAGS when testing shared linking in configure +- Add --64 option to ./configure to use -m64 with gcc +- Fix ./configure --help to correctly name options +- Have make fail if a test fails [Levin] +- Avoid buffer overrun in contrib/masmx64/gvmat64.asm [Simpson] +- Remove assembler object files from contrib + +Changes in 1.2.3.7 (24 Jan 2010) +- Always gzopen() with O_LARGEFILE if available +- Fix gzdirect() to work immediately after gzopen() or gzdopen() +- Make gzdirect() more precise when the state changes while reading +- Improve zlib.h documentation in many places +- Catch memory allocation failure in gz_open() +- Complete close operation if seek forward in gzclose_w() fails +- Return Z_ERRNO from gzclose_r() if close() fails +- Return Z_STREAM_ERROR instead of EOF for gzclose() being passed NULL +- Return zero for gzwrite() errors to match zlib.h description +- Return -1 on gzputs() error to match zlib.h description +- Add zconf.in.h to allow recovery from configure modification [Weigelt] +- Fix static library permissions in Makefile.in [Weigelt] +- Avoid warnings in configure tests that hide functionality [Weigelt] +- Add *BSD and DragonFly to Linux case in configure [gentoo 123571] +- Change libzdll.a to libz.dll.a in win32/Makefile.gcc [gentoo 288212] +- Avoid access of uninitialized data for first inflateReset2 call [Gomes] +- Keep object files in subdirectories to reduce the clutter somewhat +- Remove default Makefile and zlibdefs.h, add dummy Makefile +- Add new external functions to Z_PREFIX, remove duplicates, z_z_ -> z_ +- Remove zlibdefs.h completely -- modify zconf.h instead + +Changes in 1.2.3.6 (17 Jan 2010) +- Avoid void * arithmetic in gzread.c and gzwrite.c +- Make compilers happier with const char * for gz_error message +- Avoid unused parameter warning in inflate.c +- Avoid signed-unsigned comparison warning in inflate.c +- Indent #pragma's for traditional C +- Fix usage of strwinerror() in glib.c, change to gz_strwinerror() +- Correct email address in configure for system options +- Update make_vms.com and add make_vms.com to contrib/minizip [Zinser] +- Update zlib.map [Brown] +- Fix Makefile.in for Solaris 10 make of example64 and minizip64 [Torok] +- Apply various fixes to CMakeLists.txt [Lowman] +- Add checks on len in gzread() and gzwrite() +- Add error message for no more room for gzungetc() +- Remove zlib version check in gzwrite() +- Defer compression of gzprintf() result until need to +- Use snprintf() in gzdopen() if available +- Remove USE_MMAP configuration determination (only used by minigzip) +- Remove examples/pigz.c (available separately) +- Update examples/gun.c to 1.6 + +Changes in 1.2.3.5 (8 Jan 2010) +- Add space after #if in zutil.h for some compilers +- Fix relatively harmless bug in deflate_fast() [Exarevsky] +- Fix same problem in deflate_slow() +- Add $(SHAREDLIBV) to LIBS in Makefile.in [Brown] +- Add deflate_rle() for faster Z_RLE strategy run-length encoding +- Add deflate_huff() for faster Z_HUFFMAN_ONLY encoding +- Change name of "write" variable in inffast.c to avoid library collisions +- Fix premature EOF from gzread() in gzio.c [Brown] +- Use zlib header window size if windowBits is 0 in inflateInit2() +- Remove compressBound() call in deflate.c to avoid linking compress.o +- Replace use of errno in gz* with functions, support WinCE [Alves] +- Provide alternative to perror() in minigzip.c for WinCE [Alves] +- Don't use _vsnprintf on later versions of MSVC [Lowman] +- Add CMake build script and input file [Lowman] +- Update contrib/minizip to 1.1 [Svensson, Vollant] +- Moved nintendods directory from contrib to . +- Replace gzio.c with a new set of routines with the same functionality +- Add gzbuffer(), gzoffset(), gzclose_r(), gzclose_w() as part of above +- Update contrib/minizip to 1.1b +- Change gzeof() to return 0 on error instead of -1 to agree with zlib.h + +Changes in 1.2.3.4 (21 Dec 2009) +- Use old school .SUFFIXES in Makefile.in for FreeBSD compatibility +- Update comments in configure and Makefile.in for default --shared +- Fix test -z's in configure [Marquess] +- Build examplesh and minigzipsh when not testing +- Change NULL's to Z_NULL's in deflate.c and in comments in zlib.h +- Import LDFLAGS from the environment in configure +- Fix configure to populate SFLAGS with discovered CFLAGS options +- Adapt make_vms.com to the new Makefile.in [Zinser] +- Add zlib2ansi script for C++ compilation [Marquess] +- Add _FILE_OFFSET_BITS=64 test to make test (when applicable) +- Add AMD64 assembler code for longest match to contrib [Teterin] +- Include options from $SFLAGS when doing $LDSHARED +- Simplify 64-bit file support by introducing z_off64_t type +- Make shared object files in objs directory to work around old Sun cc +- Use only three-part version number for Darwin shared compiles +- Add rc option to ar in Makefile.in for when ./configure not run +- Add -WI,-rpath,. to LDFLAGS for OSF 1 V4* +- Set LD_LIBRARYN32_PATH for SGI IRIX shared compile +- Protect against _FILE_OFFSET_BITS being defined when compiling zlib +- Rename Makefile.in targets allstatic to static and allshared to shared +- Fix static and shared Makefile.in targets to be independent +- Correct error return bug in gz_open() by setting state [Brown] +- Put spaces before ;;'s in configure for better sh compatibility +- Add pigz.c (parallel implementation of gzip) to examples/ +- Correct constant in crc32.c to UL [Leventhal] +- Reject negative lengths in crc32_combine() +- Add inflateReset2() function to work like inflateEnd()/inflateInit2() +- Include sys/types.h for _LARGEFILE64_SOURCE [Brown] +- Correct typo in doc/algorithm.txt [Janik] +- Fix bug in adler32_combine() [Zhu] +- Catch missing-end-of-block-code error in all inflates and in puff + Assures that random input to inflate eventually results in an error +- Added enough.c (calculation of ENOUGH for inftrees.h) to examples/ +- Update ENOUGH and its usage to reflect discovered bounds +- Fix gzerror() error report on empty input file [Brown] +- Add ush casts in trees.c to avoid pedantic runtime errors +- Fix typo in zlib.h uncompress() description [Reiss] +- Correct inflate() comments with regard to automatic header detection +- Remove deprecation comment on Z_PARTIAL_FLUSH (it stays) +- Put new version of gzlog (2.0) in examples with interruption recovery +- Add puff compile option to permit invalid distance-too-far streams +- Add puff TEST command options, ability to read piped input +- Prototype the *64 functions in zlib.h when _FILE_OFFSET_BITS == 64, but + _LARGEFILE64_SOURCE not defined +- Fix Z_FULL_FLUSH to truly erase the past by resetting s->strstart +- Fix deflateSetDictionary() to use all 32K for output consistency +- Remove extraneous #define MIN_LOOKAHEAD in deflate.c (in deflate.h) +- Clear bytes after deflate lookahead to avoid use of uninitialized data +- Change a limit in inftrees.c to be more transparent to Coverity Prevent +- Update win32/zlib.def with exported symbols from zlib.h +- Correct spelling errors in zlib.h [Willem, Sobrado] +- Allow Z_BLOCK for deflate() to force a new block +- Allow negative bits in inflatePrime() to delete existing bit buffer +- Add Z_TREES flush option to inflate() to return at end of trees +- Add inflateMark() to return current state information for random access +- Add Makefile for NintendoDS to contrib [Costa] +- Add -w in configure compile tests to avoid spurious warnings [Beucler] +- Fix typos in zlib.h comments for deflateSetDictionary() +- Fix EOF detection in transparent gzread() [Maier] + +Changes in 1.2.3.3 (2 October 2006) +- Make --shared the default for configure, add a --static option +- Add compile option to permit invalid distance-too-far streams +- Add inflateUndermine() function which is required to enable above +- Remove use of "this" variable name for C++ compatibility [Marquess] +- Add testing of shared library in make test, if shared library built +- Use ftello() and fseeko() if available instead of ftell() and fseek() +- Provide two versions of all functions that use the z_off_t type for + binary compatibility -- a normal version and a 64-bit offset version, + per the Large File Support Extension when _LARGEFILE64_SOURCE is + defined; use the 64-bit versions by default when _FILE_OFFSET_BITS + is defined to be 64 +- Add a --uname= option to configure to perhaps help with cross-compiling + +Changes in 1.2.3.2 (3 September 2006) +- Turn off silly Borland warnings [Hay] +- Use off64_t and define _LARGEFILE64_SOURCE when present +- Fix missing dependency on inffixed.h in Makefile.in +- Rig configure --shared to build both shared and static [Teredesai, Truta] +- Remove zconf.in.h and instead create a new zlibdefs.h file +- Fix contrib/minizip/unzip.c non-encrypted after encrypted [Vollant] +- Add treebuild.xml (see http://treebuild.metux.de/) [Weigelt] + +Changes in 1.2.3.1 (16 August 2006) +- Add watcom directory with OpenWatcom make files [Daniel] +- Remove #undef of FAR in zconf.in.h for MVS [Fedtke] +- Update make_vms.com [Zinser] +- Use -fPIC for shared build in configure [Teredesai, Nicholson] +- Use only major version number for libz.so on IRIX and OSF1 [Reinholdtsen] +- Use fdopen() (not _fdopen()) for Interix in zutil.h [Bäck] +- Add some FAQ entries about the contrib directory +- Update the MVS question in the FAQ +- Avoid extraneous reads after EOF in gzio.c [Brown] +- Correct spelling of "successfully" in gzio.c [Randers-Pehrson] +- Add comments to zlib.h about gzerror() usage [Brown] +- Set extra flags in gzip header in gzopen() like deflate() does +- Make configure options more compatible with double-dash conventions + [Weigelt] +- Clean up compilation under Solaris SunStudio cc [Rowe, Reinholdtsen] +- Fix uninstall target in Makefile.in [Truta] +- Add pkgconfig support [Weigelt] +- Use $(DESTDIR) macro in Makefile.in [Reinholdtsen, Weigelt] +- Replace set_data_type() with a more accurate detect_data_type() in + trees.c, according to the txtvsbin.txt document [Truta] +- Swap the order of #include and #include "zlib.h" in + gzio.c, example.c and minigzip.c [Truta] +- Shut up annoying VS2005 warnings about standard C deprecation [Rowe, + Truta] (where?) +- Fix target "clean" from win32/Makefile.bor [Truta] +- Create .pdb and .manifest files in win32/makefile.msc [Ziegler, Rowe] +- Update zlib www home address in win32/DLL_FAQ.txt [Truta] +- Update contrib/masmx86/inffas32.asm for VS2005 [Vollant, Van Wassenhove] +- Enable browse info in the "Debug" and "ASM Debug" configurations in + the Visual C++ 6 project, and set (non-ASM) "Debug" as default [Truta] +- Add pkgconfig support [Weigelt] +- Add ZLIB_VER_MAJOR, ZLIB_VER_MINOR and ZLIB_VER_REVISION in zlib.h, + for use in win32/zlib1.rc [Polushin, Rowe, Truta] +- Add a document that explains the new text detection scheme to + doc/txtvsbin.txt [Truta] +- Add rfc1950.txt, rfc1951.txt and rfc1952.txt to doc/ [Truta] +- Move algorithm.txt into doc/ [Truta] +- Synchronize FAQ with website +- Fix compressBound(), was low for some pathological cases [Fearnley] +- Take into account wrapper variations in deflateBound() +- Set examples/zpipe.c input and output to binary mode for Windows +- Update examples/zlib_how.html with new zpipe.c (also web site) +- Fix some warnings in examples/gzlog.c and examples/zran.c (it seems + that gcc became pickier in 4.0) +- Add zlib.map for Linux: "All symbols from zlib-1.1.4 remain + un-versioned, the patch adds versioning only for symbols introduced in + zlib-1.2.0 or later. It also declares as local those symbols which are + not designed to be exported." [Levin] +- Update Z_PREFIX list in zconf.in.h, add --zprefix option to configure +- Do not initialize global static by default in trees.c, add a response + NO_INIT_GLOBAL_POINTERS to initialize them if needed [Marquess] +- Don't use strerror() in gzio.c under WinCE [Yakimov] +- Don't use errno.h in zutil.h under WinCE [Yakimov] +- Move arguments for AR to its usage to allow replacing ar [Marot] +- Add HAVE_VISIBILITY_PRAGMA in zconf.in.h for Mozilla [Randers-Pehrson] +- Improve inflateInit() and inflateInit2() documentation +- Fix structure size comment in inflate.h +- Change configure help option from --h* to --help [Santos] + +Changes in 1.2.3 (18 July 2005) +- Apply security vulnerability fixes to contrib/infback9 as well +- Clean up some text files (carriage returns, trailing space) +- Update testzlib, vstudio, masmx64, and masmx86 in contrib [Vollant] + +Changes in 1.2.2.4 (11 July 2005) +- Add inflatePrime() function for starting inflation at bit boundary +- Avoid some Visual C warnings in deflate.c +- Avoid more silly Visual C warnings in inflate.c and inftrees.c for 64-bit + compile +- Fix some spelling errors in comments [Betts] +- Correct inflateInit2() error return documentation in zlib.h +- Add zran.c example of compressed data random access to examples + directory, shows use of inflatePrime() +- Fix cast for assignments to strm->state in inflate.c and infback.c +- Fix zlibCompileFlags() in zutil.c to use 1L for long shifts [Oberhumer] +- Move declarations of gf2 functions to right place in crc32.c [Oberhumer] +- Add cast in trees.c t avoid a warning [Oberhumer] +- Avoid some warnings in fitblk.c, gun.c, gzjoin.c in examples [Oberhumer] +- Update make_vms.com [Zinser] +- Initialize state->write in inflateReset() since copied in inflate_fast() +- Be more strict on incomplete code sets in inflate_table() and increase + ENOUGH and MAXD -- this repairs a possible security vulnerability for + invalid inflate input. Thanks to Tavis Ormandy and Markus Oberhumer for + discovering the vulnerability and providing test cases. +- Add ia64 support to configure for HP-UX [Smith] +- Add error return to gzread() for format or i/o error [Levin] +- Use malloc.h for OS/2 [Necasek] + +Changes in 1.2.2.3 (27 May 2005) +- Replace 1U constants in inflate.c and inftrees.c for 64-bit compile +- Typecast fread() return values in gzio.c [Vollant] +- Remove trailing space in minigzip.c outmode (VC++ can't deal with it) +- Fix crc check bug in gzread() after gzungetc() [Heiner] +- Add the deflateTune() function to adjust internal compression parameters +- Add a fast gzip decompressor, gun.c, to examples (use of inflateBack) +- Remove an incorrect assertion in examples/zpipe.c +- Add C++ wrapper in infback9.h [Donais] +- Fix bug in inflateCopy() when decoding fixed codes +- Note in zlib.h how much deflateSetDictionary() actually uses +- Remove USE_DICT_HEAD in deflate.c (would mess up inflate if used) +- Add _WIN32_WCE to define WIN32 in zconf.in.h [Spencer] +- Don't include stderr.h or errno.h for _WIN32_WCE in zutil.h [Spencer] +- Add gzdirect() function to indicate transparent reads +- Update contrib/minizip [Vollant] +- Fix compilation of deflate.c when both ASMV and FASTEST [Oberhumer] +- Add casts in crc32.c to avoid warnings [Oberhumer] +- Add contrib/masmx64 [Vollant] +- Update contrib/asm586, asm686, masmx86, testzlib, vstudio [Vollant] + +Changes in 1.2.2.2 (30 December 2004) +- Replace structure assignments in deflate.c and inflate.c with zmemcpy to + avoid implicit memcpy calls (portability for no-library compilation) +- Increase sprintf() buffer size in gzdopen() to allow for large numbers +- Add INFLATE_STRICT to check distances against zlib header +- Improve WinCE errno handling and comments [Chang] +- Remove comment about no gzip header processing in FAQ +- Add Z_FIXED strategy option to deflateInit2() to force fixed trees +- Add updated make_vms.com [Coghlan], update README +- Create a new "examples" directory, move gzappend.c there, add zpipe.c, + fitblk.c, gzlog.[ch], gzjoin.c, and zlib_how.html. +- Add FAQ entry and comments in deflate.c on uninitialized memory access +- Add Solaris 9 make options in configure [Gilbert] +- Allow strerror() usage in gzio.c for STDC +- Fix DecompressBuf in contrib/delphi/ZLib.pas [ManChesTer] +- Update contrib/masmx86/inffas32.asm and gvmat32.asm [Vollant] +- Use z_off_t for adler32_combine() and crc32_combine() lengths +- Make adler32() much faster for small len +- Use OS_CODE in deflate() default gzip header + +Changes in 1.2.2.1 (31 October 2004) +- Allow inflateSetDictionary() call for raw inflate +- Fix inflate header crc check bug for file names and comments +- Add deflateSetHeader() and gz_header structure for custom gzip headers +- Add inflateGetheader() to retrieve gzip headers +- Add crc32_combine() and adler32_combine() functions +- Add alloc_func, free_func, in_func, out_func to Z_PREFIX list +- Use zstreamp consistently in zlib.h (inflate_back functions) +- Remove GUNZIP condition from definition of inflate_mode in inflate.h + and in contrib/inflate86/inffast.S [Truta, Anderson] +- Add support for AMD64 in contrib/inflate86/inffas86.c [Anderson] +- Update projects/README.projects and projects/visualc6 [Truta] +- Update win32/DLL_FAQ.txt [Truta] +- Avoid warning under NO_GZCOMPRESS in gzio.c; fix typo [Truta] +- Deprecate Z_ASCII; use Z_TEXT instead [Truta] +- Use a new algorithm for setting strm->data_type in trees.c [Truta] +- Do not define an exit() prototype in zutil.c unless DEBUG defined +- Remove prototype of exit() from zutil.c, example.c, minigzip.c [Truta] +- Add comment in zlib.h for Z_NO_FLUSH parameter to deflate() +- Fix Darwin build version identification [Peterson] + +Changes in 1.2.2 (3 October 2004) +- Update zlib.h comments on gzip in-memory processing +- Set adler to 1 in inflateReset() to support Java test suite [Walles] +- Add contrib/dotzlib [Ravn] +- Update win32/DLL_FAQ.txt [Truta] +- Update contrib/minizip [Vollant] +- Move contrib/visual-basic.txt to old/ [Truta] +- Fix assembler builds in projects/visualc6/ [Truta] + +Changes in 1.2.1.2 (9 September 2004) +- Update INDEX file +- Fix trees.c to update strm->data_type (no one ever noticed!) +- Fix bug in error case in inflate.c, infback.c, and infback9.c [Brown] +- Add "volatile" to crc table flag declaration (for DYNAMIC_CRC_TABLE) +- Add limited multitasking protection to DYNAMIC_CRC_TABLE +- Add NO_vsnprintf for VMS in zutil.h [Mozilla] +- Don't declare strerror() under VMS [Mozilla] +- Add comment to DYNAMIC_CRC_TABLE to use get_crc_table() to initialize +- Update contrib/ada [Anisimkov] +- Update contrib/minizip [Vollant] +- Fix configure to not hardcode directories for Darwin [Peterson] +- Fix gzio.c to not return error on empty files [Brown] +- Fix indentation; update version in contrib/delphi/ZLib.pas and + contrib/pascal/zlibpas.pas [Truta] +- Update mkasm.bat in contrib/masmx86 [Truta] +- Update contrib/untgz [Truta] +- Add projects/README.projects [Truta] +- Add project for MS Visual C++ 6.0 in projects/visualc6 [Cadieux, Truta] +- Update win32/DLL_FAQ.txt [Truta] +- Update list of Z_PREFIX symbols in zconf.h [Randers-Pehrson, Truta] +- Remove an unnecessary assignment to curr in inftrees.c [Truta] +- Add OS/2 to exe builds in configure [Poltorak] +- Remove err dummy parameter in zlib.h [Kientzle] + +Changes in 1.2.1.1 (9 January 2004) +- Update email address in README +- Several FAQ updates +- Fix a big fat bug in inftrees.c that prevented decoding valid + dynamic blocks with only literals and no distance codes -- + Thanks to "Hot Emu" for the bug report and sample file +- Add a note to puff.c on no distance codes case. + +Changes in 1.2.1 (17 November 2003) +- Remove a tab in contrib/gzappend/gzappend.c +- Update some interfaces in contrib for new zlib functions +- Update zlib version number in some contrib entries +- Add Windows CE definition for ptrdiff_t in zutil.h [Mai, Truta] +- Support shared libraries on Hurd and KFreeBSD [Brown] +- Fix error in NO_DIVIDE option of adler32.c + +Changes in 1.2.0.8 (4 November 2003) +- Update version in contrib/delphi/ZLib.pas and contrib/pascal/zlibpas.pas +- Add experimental NO_DIVIDE #define in adler32.c + - Possibly faster on some processors (let me know if it is) +- Correct Z_BLOCK to not return on first inflate call if no wrap +- Fix strm->data_type on inflate() return to correctly indicate EOB +- Add deflatePrime() function for appending in the middle of a byte +- Add contrib/gzappend for an example of appending to a stream +- Update win32/DLL_FAQ.txt [Truta] +- Delete Turbo C comment in README [Truta] +- Improve some indentation in zconf.h [Truta] +- Fix infinite loop on bad input in configure script [Church] +- Fix gzeof() for concatenated gzip files [Johnson] +- Add example to contrib/visual-basic.txt [Michael B.] +- Add -p to mkdir's in Makefile.in [vda] +- Fix configure to properly detect presence or lack of printf functions +- Add AS400 support [Monnerat] +- Add a little Cygwin support [Wilson] + +Changes in 1.2.0.7 (21 September 2003) +- Correct some debug formats in contrib/infback9 +- Cast a type in a debug statement in trees.c +- Change search and replace delimiter in configure from % to # [Beebe] +- Update contrib/untgz to 0.2 with various fixes [Truta] +- Add build support for Amiga [Nikl] +- Remove some directories in old that have been updated to 1.2 +- Add dylib building for Mac OS X in configure and Makefile.in +- Remove old distribution stuff from Makefile +- Update README to point to DLL_FAQ.txt, and add comment on Mac OS X +- Update links in README + +Changes in 1.2.0.6 (13 September 2003) +- Minor FAQ updates +- Update contrib/minizip to 1.00 [Vollant] +- Remove test of gz functions in example.c when GZ_COMPRESS defined [Truta] +- Update POSTINC comment for 68060 [Nikl] +- Add contrib/infback9 with deflate64 decoding (unsupported) +- For MVS define NO_vsnprintf and undefine FAR [van Burik] +- Add pragma for fdopen on MVS [van Burik] + +Changes in 1.2.0.5 (8 September 2003) +- Add OF to inflateBackEnd() declaration in zlib.h +- Remember start when using gzdopen in the middle of a file +- Use internal off_t counters in gz* functions to properly handle seeks +- Perform more rigorous check for distance-too-far in inffast.c +- Add Z_BLOCK flush option to return from inflate at block boundary +- Set strm->data_type on return from inflate + - Indicate bits unused, if at block boundary, and if in last block +- Replace size_t with ptrdiff_t in crc32.c, and check for correct size +- Add condition so old NO_DEFLATE define still works for compatibility +- FAQ update regarding the Windows DLL [Truta] +- INDEX update: add qnx entry, remove aix entry [Truta] +- Install zlib.3 into mandir [Wilson] +- Move contrib/zlib_dll_FAQ.txt to win32/DLL_FAQ.txt; update [Truta] +- Adapt the zlib interface to the new DLL convention guidelines [Truta] +- Introduce ZLIB_WINAPI macro to allow the export of functions using + the WINAPI calling convention, for Visual Basic [Vollant, Truta] +- Update msdos and win32 scripts and makefiles [Truta] +- Export symbols by name, not by ordinal, in win32/zlib.def [Truta] +- Add contrib/ada [Anisimkov] +- Move asm files from contrib/vstudio/vc70_32 to contrib/asm386 [Truta] +- Rename contrib/asm386 to contrib/masmx86 [Truta, Vollant] +- Add contrib/masm686 [Truta] +- Fix offsets in contrib/inflate86 and contrib/masmx86/inffas32.asm + [Truta, Vollant] +- Update contrib/delphi; rename to contrib/pascal; add example [Truta] +- Remove contrib/delphi2; add a new contrib/delphi [Truta] +- Avoid inclusion of the nonstandard in contrib/iostream, + and fix some method prototypes [Truta] +- Fix the ZCR_SEED2 constant to avoid warnings in contrib/minizip + [Truta] +- Avoid the use of backslash (\) in contrib/minizip [Vollant] +- Fix file time handling in contrib/untgz; update makefiles [Truta] +- Update contrib/vstudio/vc70_32 to comply with the new DLL guidelines + [Vollant] +- Remove contrib/vstudio/vc15_16 [Vollant] +- Rename contrib/vstudio/vc70_32 to contrib/vstudio/vc7 [Truta] +- Update README.contrib [Truta] +- Invert the assignment order of match_head and s->prev[...] in + INSERT_STRING [Truta] +- Compare TOO_FAR with 32767 instead of 32768, to avoid 16-bit warnings + [Truta] +- Compare function pointers with 0, not with NULL or Z_NULL [Truta] +- Fix prototype of syncsearch in inflate.c [Truta] +- Introduce ASMINF macro to be enabled when using an ASM implementation + of inflate_fast [Truta] +- Change NO_DEFLATE to NO_GZCOMPRESS [Truta] +- Modify test_gzio in example.c to take a single file name as a + parameter [Truta] +- Exit the example.c program if gzopen fails [Truta] +- Add type casts around strlen in example.c [Truta] +- Remove casting to sizeof in minigzip.c; give a proper type + to the variable compared with SUFFIX_LEN [Truta] +- Update definitions of STDC and STDC99 in zconf.h [Truta] +- Synchronize zconf.h with the new Windows DLL interface [Truta] +- Use SYS16BIT instead of __32BIT__ to distinguish between + 16- and 32-bit platforms [Truta] +- Use far memory allocators in small 16-bit memory models for + Turbo C [Truta] +- Add info about the use of ASMV, ASMINF and ZLIB_WINAPI in + zlibCompileFlags [Truta] +- Cygwin has vsnprintf [Wilson] +- In Windows16, OS_CODE is 0, as in MSDOS [Truta] +- In Cygwin, OS_CODE is 3 (Unix), not 11 (Windows32) [Wilson] + +Changes in 1.2.0.4 (10 August 2003) +- Minor FAQ updates +- Be more strict when checking inflateInit2's windowBits parameter +- Change NO_GUNZIP compile option to NO_GZIP to cover deflate as well +- Add gzip wrapper option to deflateInit2 using windowBits +- Add updated QNX rule in configure and qnx directory [Bonnefoy] +- Make inflate distance-too-far checks more rigorous +- Clean up FAR usage in inflate +- Add casting to sizeof() in gzio.c and minigzip.c + +Changes in 1.2.0.3 (19 July 2003) +- Fix silly error in gzungetc() implementation [Vollant] +- Update contrib/minizip and contrib/vstudio [Vollant] +- Fix printf format in example.c +- Correct cdecl support in zconf.in.h [Anisimkov] +- Minor FAQ updates + +Changes in 1.2.0.2 (13 July 2003) +- Add ZLIB_VERNUM in zlib.h for numerical preprocessor comparisons +- Attempt to avoid warnings in crc32.c for pointer-int conversion +- Add AIX to configure, remove aix directory [Bakker] +- Add some casts to minigzip.c +- Improve checking after insecure sprintf() or vsprintf() calls +- Remove #elif's from crc32.c +- Change leave label to inf_leave in inflate.c and infback.c to avoid + library conflicts +- Remove inflate gzip decoding by default--only enable gzip decoding by + special request for stricter backward compatibility +- Add zlibCompileFlags() function to return compilation information +- More typecasting in deflate.c to avoid warnings +- Remove leading underscore from _Capital #defines [Truta] +- Fix configure to link shared library when testing +- Add some Windows CE target adjustments [Mai] +- Remove #define ZLIB_DLL in zconf.h [Vollant] +- Add zlib.3 [Rodgers] +- Update RFC URL in deflate.c and algorithm.txt [Mai] +- Add zlib_dll_FAQ.txt to contrib [Truta] +- Add UL to some constants [Truta] +- Update minizip and vstudio [Vollant] +- Remove vestigial NEED_DUMMY_RETURN from zconf.in.h +- Expand use of NO_DUMMY_DECL to avoid all dummy structures +- Added iostream3 to contrib [Schwardt] +- Replace rewind() with fseek() for WinCE [Truta] +- Improve setting of zlib format compression level flags + - Report 0 for huffman and rle strategies and for level == 0 or 1 + - Report 2 only for level == 6 +- Only deal with 64K limit when necessary at compile time [Truta] +- Allow TOO_FAR check to be turned off at compile time [Truta] +- Add gzclearerr() function [Souza] +- Add gzungetc() function + +Changes in 1.2.0.1 (17 March 2003) +- Add Z_RLE strategy for run-length encoding [Truta] + - When Z_RLE requested, restrict matches to distance one + - Update zlib.h, minigzip.c, gzopen(), gzdopen() for Z_RLE +- Correct FASTEST compilation to allow level == 0 +- Clean up what gets compiled for FASTEST +- Incorporate changes to zconf.in.h [Vollant] + - Refine detection of Turbo C need for dummy returns + - Refine ZLIB_DLL compilation + - Include additional header file on VMS for off_t typedef +- Try to use _vsnprintf where it supplants vsprintf [Vollant] +- Add some casts in inffast.c +- Enchance comments in zlib.h on what happens if gzprintf() tries to + write more than 4095 bytes before compression +- Remove unused state from inflateBackEnd() +- Remove exit(0) from minigzip.c, example.c +- Get rid of all those darn tabs +- Add "check" target to Makefile.in that does the same thing as "test" +- Add "mostlyclean" and "maintainer-clean" targets to Makefile.in +- Update contrib/inflate86 [Anderson] +- Update contrib/testzlib, contrib/vstudio, contrib/minizip [Vollant] +- Add msdos and win32 directories with makefiles [Truta] +- More additions and improvements to the FAQ + +Changes in 1.2.0 (9 March 2003) +- New and improved inflate code + - About 20% faster + - Does not allocate 32K window unless and until needed + - Automatically detects and decompresses gzip streams + - Raw inflate no longer needs an extra dummy byte at end + - Added inflateBack functions using a callback interface--even faster + than inflate, useful for file utilities (gzip, zip) + - Added inflateCopy() function to record state for random access on + externally generated deflate streams (e.g. in gzip files) + - More readable code (I hope) +- New and improved crc32() + - About 50% faster, thanks to suggestions from Rodney Brown +- Add deflateBound() and compressBound() functions +- Fix memory leak in deflateInit2() +- Permit setting dictionary for raw deflate (for parallel deflate) +- Fix const declaration for gzwrite() +- Check for some malloc() failures in gzio.c +- Fix bug in gzopen() on single-byte file 0x1f +- Fix bug in gzread() on concatenated file with 0x1f at end of buffer + and next buffer doesn't start with 0x8b +- Fix uncompress() to return Z_DATA_ERROR on truncated input +- Free memory at end of example.c +- Remove MAX #define in trees.c (conflicted with some libraries) +- Fix static const's in deflate.c, gzio.c, and zutil.[ch] +- Declare malloc() and free() in gzio.c if STDC not defined +- Use malloc() instead of calloc() in zutil.c if int big enough +- Define STDC for AIX +- Add aix/ with approach for compiling shared library on AIX +- Add HP-UX support for shared libraries in configure +- Add OpenUNIX support for shared libraries in configure +- Use $cc instead of gcc to build shared library +- Make prefix directory if needed when installing +- Correct Macintosh avoidance of typedef Byte in zconf.h +- Correct Turbo C memory allocation when under Linux +- Use libz.a instead of -lz in Makefile (assure use of compiled library) +- Update configure to check for snprintf or vsnprintf functions and their + return value, warn during make if using an insecure function +- Fix configure problem with compile-time knowledge of HAVE_UNISTD_H that + is lost when library is used--resolution is to build new zconf.h +- Documentation improvements (in zlib.h): + - Document raw deflate and inflate + - Update RFCs URL + - Point out that zlib and gzip formats are different + - Note that Z_BUF_ERROR is not fatal + - Document string limit for gzprintf() and possible buffer overflow + - Note requirement on avail_out when flushing + - Note permitted values of flush parameter of inflate() +- Add some FAQs (and even answers) to the FAQ +- Add contrib/inflate86/ for x86 faster inflate +- Add contrib/blast/ for PKWare Data Compression Library decompression +- Add contrib/puff/ simple inflate for deflate format description + +Changes in 1.1.4 (11 March 2002) +- ZFREE was repeated on same allocation on some error conditions. + This creates a security problem described in + http://www.zlib.org/advisory-2002-03-11.txt +- Returned incorrect error (Z_MEM_ERROR) on some invalid data +- Avoid accesses before window for invalid distances with inflate window + less than 32K. +- force windowBits > 8 to avoid a bug in the encoder for a window size + of 256 bytes. (A complete fix will be available in 1.1.5). + +Changes in 1.1.3 (9 July 1998) +- fix "an inflate input buffer bug that shows up on rare but persistent + occasions" (Mark) +- fix gzread and gztell for concatenated .gz files (Didier Le Botlan) +- fix gzseek(..., SEEK_SET) in write mode +- fix crc check after a gzeek (Frank Faubert) +- fix miniunzip when the last entry in a zip file is itself a zip file + (J Lillge) +- add contrib/asm586 and contrib/asm686 (Brian Raiter) + See http://www.muppetlabs.com/~breadbox/software/assembly.html +- add support for Delphi 3 in contrib/delphi (Bob Dellaca) +- add support for C++Builder 3 and Delphi 3 in contrib/delphi2 (Davide Moretti) +- do not exit prematurely in untgz if 0 at start of block (Magnus Holmgren) +- use macro EXTERN instead of extern to support DLL for BeOS (Sander Stoks) +- added a FAQ file + +- Support gzdopen on Mac with Metrowerks (Jason Linhart) +- Do not redefine Byte on Mac (Brad Pettit & Jason Linhart) +- define SEEK_END too if SEEK_SET is not defined (Albert Chin-A-Young) +- avoid some warnings with Borland C (Tom Tanner) +- fix a problem in contrib/minizip/zip.c for 16-bit MSDOS (Gilles Vollant) +- emulate utime() for WIN32 in contrib/untgz (Gilles Vollant) +- allow several arguments to configure (Tim Mooney, Frodo Looijaard) +- use libdir and includedir in Makefile.in (Tim Mooney) +- support shared libraries on OSF1 V4 (Tim Mooney) +- remove so_locations in "make clean" (Tim Mooney) +- fix maketree.c compilation error (Glenn, Mark) +- Python interface to zlib now in Python 1.5 (Jeremy Hylton) +- new Makefile.riscos (Rich Walker) +- initialize static descriptors in trees.c for embedded targets (Nick Smith) +- use "foo-gz" in example.c for RISCOS and VMS (Nick Smith) +- add the OS/2 files in Makefile.in too (Andrew Zabolotny) +- fix fdopen and halloc macros for Microsoft C 6.0 (Tom Lane) +- fix maketree.c to allow clean compilation of inffixed.h (Mark) +- fix parameter check in deflateCopy (Gunther Nikl) +- cleanup trees.c, use compressed_len only in debug mode (Christian Spieler) +- Many portability patches by Christian Spieler: + . zutil.c, zutil.h: added "const" for zmem* + . Make_vms.com: fixed some typos + . Make_vms.com: msdos/Makefile.*: removed zutil.h from some dependency lists + . msdos/Makefile.msc: remove "default rtl link library" info from obj files + . msdos/Makefile.*: use model-dependent name for the built zlib library + . msdos/Makefile.emx, nt/Makefile.emx, nt/Makefile.gcc: + new makefiles, for emx (DOS/OS2), emx&rsxnt and mingw32 (Windows 9x / NT) +- use define instead of typedef for Bytef also for MSC small/medium (Tom Lane) +- replace __far with _far for better portability (Christian Spieler, Tom Lane) +- fix test for errno.h in configure (Tim Newsham) + +Changes in 1.1.2 (19 March 98) +- added contrib/minzip, mini zip and unzip based on zlib (Gilles Vollant) + See http://www.winimage.com/zLibDll/unzip.html +- preinitialize the inflate tables for fixed codes, to make the code + completely thread safe (Mark) +- some simplifications and slight speed-up to the inflate code (Mark) +- fix gzeof on non-compressed files (Allan Schrum) +- add -std1 option in configure for OSF1 to fix gzprintf (Martin Mokrejs) +- use default value of 4K for Z_BUFSIZE for 16-bit MSDOS (Tim Wegner + Glenn) +- added os2/Makefile.def and os2/zlib.def (Andrew Zabolotny) +- add shared lib support for UNIX_SV4.2MP (MATSUURA Takanori) +- do not wrap extern "C" around system includes (Tom Lane) +- mention zlib binding for TCL in README (Andreas Kupries) +- added amiga/Makefile.pup for Amiga powerUP SAS/C PPC (Andreas Kleinert) +- allow "make install prefix=..." even after configure (Glenn Randers-Pehrson) +- allow "configure --prefix $HOME" (Tim Mooney) +- remove warnings in example.c and gzio.c (Glenn Randers-Pehrson) +- move Makefile.sas to amiga/Makefile.sas + +Changes in 1.1.1 (27 Feb 98) +- fix macros _tr_tally_* in deflate.h for debug mode (Glenn Randers-Pehrson) +- remove block truncation heuristic which had very marginal effect for zlib + (smaller lit_bufsize than in gzip 1.2.4) and degraded a little the + compression ratio on some files. This also allows inlining _tr_tally for + matches in deflate_slow. +- added msdos/Makefile.w32 for WIN32 Microsoft Visual C++ (Bob Frazier) + +Changes in 1.1.0 (24 Feb 98) +- do not return STREAM_END prematurely in inflate (John Bowler) +- revert to the zlib 1.0.8 inflate to avoid the gcc 2.8.0 bug (Jeremy Buhler) +- compile with -DFASTEST to get compression code optimized for speed only +- in minigzip, try mmap'ing the input file first (Miguel Albrecht) +- increase size of I/O buffers in minigzip.c and gzio.c (not a big gain + on Sun but significant on HP) + +- add a pointer to experimental unzip library in README (Gilles Vollant) +- initialize variable gcc in configure (Chris Herborth) + +Changes in 1.0.9 (17 Feb 1998) +- added gzputs and gzgets functions +- do not clear eof flag in gzseek (Mark Diekhans) +- fix gzseek for files in transparent mode (Mark Diekhans) +- do not assume that vsprintf returns the number of bytes written (Jens Krinke) +- replace EXPORT with ZEXPORT to avoid conflict with other programs +- added compress2 in zconf.h, zlib.def, zlib.dnt +- new asm code from Gilles Vollant in contrib/asm386 +- simplify the inflate code (Mark): + . Replace ZALLOC's in huft_build() with single ZALLOC in inflate_blocks_new() + . ZALLOC the length list in inflate_trees_fixed() instead of using stack + . ZALLOC the value area for huft_build() instead of using stack + . Simplify Z_FINISH check in inflate() + +- Avoid gcc 2.8.0 comparison bug a little differently than zlib 1.0.8 +- in inftrees.c, avoid cc -O bug on HP (Farshid Elahi) +- in zconf.h move the ZLIB_DLL stuff earlier to avoid problems with + the declaration of FAR (Gilles VOllant) +- install libz.so* with mode 755 (executable) instead of 644 (Marc Lehmann) +- read_buf buf parameter of type Bytef* instead of charf* +- zmemcpy parameters are of type Bytef*, not charf* (Joseph Strout) +- do not redeclare unlink in minigzip.c for WIN32 (John Bowler) +- fix check for presence of directories in "make install" (Ian Willis) + +Changes in 1.0.8 (27 Jan 1998) +- fixed offsets in contrib/asm386/gvmat32.asm (Gilles Vollant) +- fix gzgetc and gzputc for big endian systems (Markus Oberhumer) +- added compress2() to allow setting the compression level +- include sys/types.h to get off_t on some systems (Marc Lehmann & QingLong) +- use constant arrays for the static trees in trees.c instead of computing + them at run time (thanks to Ken Raeburn for this suggestion). To create + trees.h, compile with GEN_TREES_H and run "make test". +- check return code of example in "make test" and display result +- pass minigzip command line options to file_compress +- simplifying code of inflateSync to avoid gcc 2.8 bug + +- support CC="gcc -Wall" in configure -s (QingLong) +- avoid a flush caused by ftell in gzopen for write mode (Ken Raeburn) +- fix test for shared library support to avoid compiler warnings +- zlib.lib -> zlib.dll in msdos/zlib.rc (Gilles Vollant) +- check for TARGET_OS_MAC in addition to MACOS (Brad Pettit) +- do not use fdopen for Metrowerks on Mac (Brad Pettit)) +- add checks for gzputc and gzputc in example.c +- avoid warnings in gzio.c and deflate.c (Andreas Kleinert) +- use const for the CRC table (Ken Raeburn) +- fixed "make uninstall" for shared libraries +- use Tracev instead of Trace in infblock.c +- in example.c use correct compressed length for test_sync +- suppress +vnocompatwarnings in configure for HPUX (not always supported) + +Changes in 1.0.7 (20 Jan 1998) +- fix gzseek which was broken in write mode +- return error for gzseek to negative absolute position +- fix configure for Linux (Chun-Chung Chen) +- increase stack space for MSC (Tim Wegner) +- get_crc_table and inflateSyncPoint are EXPORTed (Gilles Vollant) +- define EXPORTVA for gzprintf (Gilles Vollant) +- added man page zlib.3 (Rick Rodgers) +- for contrib/untgz, fix makedir() and improve Makefile + +- check gzseek in write mode in example.c +- allocate extra buffer for seeks only if gzseek is actually called +- avoid signed/unsigned comparisons (Tim Wegner, Gilles Vollant) +- add inflateSyncPoint in zconf.h +- fix list of exported functions in nt/zlib.dnt and mdsos/zlib.def + +Changes in 1.0.6 (19 Jan 1998) +- add functions gzprintf, gzputc, gzgetc, gztell, gzeof, gzseek, gzrewind and + gzsetparams (thanks to Roland Giersig and Kevin Ruland for some of this code) +- Fix a deflate bug occurring only with compression level 0 (thanks to + Andy Buckler for finding this one). +- In minigzip, pass transparently also the first byte for .Z files. +- return Z_BUF_ERROR instead of Z_OK if output buffer full in uncompress() +- check Z_FINISH in inflate (thanks to Marc Schluper) +- Implement deflateCopy (thanks to Adam Costello) +- make static libraries by default in configure, add --shared option. +- move MSDOS or Windows specific files to directory msdos +- suppress the notion of partial flush to simplify the interface + (but the symbol Z_PARTIAL_FLUSH is kept for compatibility with 1.0.4) +- suppress history buffer provided by application to simplify the interface + (this feature was not implemented anyway in 1.0.4) +- next_in and avail_in must be initialized before calling inflateInit or + inflateInit2 +- add EXPORT in all exported functions (for Windows DLL) +- added Makefile.nt (thanks to Stephen Williams) +- added the unsupported "contrib" directory: + contrib/asm386/ by Gilles Vollant + 386 asm code replacing longest_match(). + contrib/iostream/ by Kevin Ruland + A C++ I/O streams interface to the zlib gz* functions + contrib/iostream2/ by Tyge Løvset + Another C++ I/O streams interface + contrib/untgz/ by "Pedro A. Aranda Guti\irrez" + A very simple tar.gz file extractor using zlib + contrib/visual-basic.txt by Carlos Rios + How to use compress(), uncompress() and the gz* functions from VB. +- pass params -f (filtered data), -h (huffman only), -1 to -9 (compression + level) in minigzip (thanks to Tom Lane) + +- use const for rommable constants in deflate +- added test for gzseek and gztell in example.c +- add undocumented function inflateSyncPoint() (hack for Paul Mackerras) +- add undocumented function zError to convert error code to string + (for Tim Smithers) +- Allow compilation of gzio with -DNO_DEFLATE to avoid the compression code. +- Use default memcpy for Symantec MSDOS compiler. +- Add EXPORT keyword for check_func (needed for Windows DLL) +- add current directory to LD_LIBRARY_PATH for "make test" +- create also a link for libz.so.1 +- added support for FUJITSU UXP/DS (thanks to Toshiaki Nomura) +- use $(SHAREDLIB) instead of libz.so in Makefile.in (for HPUX) +- added -soname for Linux in configure (Chun-Chung Chen, +- assign numbers to the exported functions in zlib.def (for Windows DLL) +- add advice in zlib.h for best usage of deflateSetDictionary +- work around compiler bug on Atari (cast Z_NULL in call of s->checkfn) +- allow compilation with ANSI keywords only enabled for TurboC in large model +- avoid "versionString"[0] (Borland bug) +- add NEED_DUMMY_RETURN for Borland +- use variable z_verbose for tracing in debug mode (L. Peter Deutsch). +- allow compilation with CC +- defined STDC for OS/2 (David Charlap) +- limit external names to 8 chars for MVS (Thomas Lund) +- in minigzip.c, use static buffers only for 16-bit systems +- fix suffix check for "minigzip -d foo.gz" +- do not return an error for the 2nd of two consecutive gzflush() (Felix Lee) +- use _fdopen instead of fdopen for MSC >= 6.0 (Thomas Fanslau) +- added makelcc.bat for lcc-win32 (Tom St Denis) +- in Makefile.dj2, use copy and del instead of install and rm (Frank Donahoe) +- Avoid expanded $Id$. Use "rcs -kb" or "cvs admin -kb" to avoid Id expansion. +- check for unistd.h in configure (for off_t) +- remove useless check parameter in inflate_blocks_free +- avoid useless assignment of s->check to itself in inflate_blocks_new +- do not flush twice in gzclose (thanks to Ken Raeburn) +- rename FOPEN as F_OPEN to avoid clash with /usr/include/sys/file.h +- use NO_ERRNO_H instead of enumeration of operating systems with errno.h +- work around buggy fclose on pipes for HP/UX +- support zlib DLL with BORLAND C++ 5.0 (thanks to Glenn Randers-Pehrson) +- fix configure if CC is already equal to gcc + +Changes in 1.0.5 (3 Jan 98) +- Fix inflate to terminate gracefully when fed corrupted or invalid data +- Use const for rommable constants in inflate +- Eliminate memory leaks on error conditions in inflate +- Removed some vestigial code in inflate +- Update web address in README + +Changes in 1.0.4 (24 Jul 96) +- In very rare conditions, deflate(s, Z_FINISH) could fail to produce an EOF + bit, so the decompressor could decompress all the correct data but went + on to attempt decompressing extra garbage data. This affected minigzip too. +- zlibVersion and gzerror return const char* (needed for DLL) +- port to RISCOS (no fdopen, no multiple dots, no unlink, no fileno) +- use z_error only for DEBUG (avoid problem with DLLs) + +Changes in 1.0.3 (2 Jul 96) +- use z_streamp instead of z_stream *, which is now a far pointer in MSDOS + small and medium models; this makes the library incompatible with previous + versions for these models. (No effect in large model or on other systems.) +- return OK instead of BUF_ERROR if previous deflate call returned with + avail_out as zero but there is nothing to do +- added memcmp for non STDC compilers +- define NO_DUMMY_DECL for more Mac compilers (.h files merged incorrectly) +- define __32BIT__ if __386__ or i386 is defined (pb. with Watcom and SCO) +- better check for 16-bit mode MSC (avoids problem with Symantec) + +Changes in 1.0.2 (23 May 96) +- added Windows DLL support +- added a function zlibVersion (for the DLL support) +- fixed declarations using Bytef in infutil.c (pb with MSDOS medium model) +- Bytef is define's instead of typedef'd only for Borland C +- avoid reading uninitialized memory in example.c +- mention in README that the zlib format is now RFC1950 +- updated Makefile.dj2 +- added algorithm.doc + +Changes in 1.0.1 (20 May 96) [1.0 skipped to avoid confusion] +- fix array overlay in deflate.c which sometimes caused bad compressed data +- fix inflate bug with empty stored block +- fix MSDOS medium model which was broken in 0.99 +- fix deflateParams() which could generate bad compressed data. +- Bytef is define'd instead of typedef'ed (work around Borland bug) +- added an INDEX file +- new makefiles for DJGPP (Makefile.dj2), 32-bit Borland (Makefile.b32), + Watcom (Makefile.wat), Amiga SAS/C (Makefile.sas) +- speed up adler32 for modern machines without auto-increment +- added -ansi for IRIX in configure +- static_init_done in trees.c is an int +- define unlink as delete for VMS +- fix configure for QNX +- add configure branch for SCO and HPUX +- avoid many warnings (unused variables, dead assignments, etc...) +- no fdopen for BeOS +- fix the Watcom fix for 32 bit mode (define FAR as empty) +- removed redefinition of Byte for MKWERKS +- work around an MWKERKS bug (incorrect merge of all .h files) + +Changes in 0.99 (27 Jan 96) +- allow preset dictionary shared between compressor and decompressor +- allow compression level 0 (no compression) +- add deflateParams in zlib.h: allow dynamic change of compression level + and compression strategy. +- test large buffers and deflateParams in example.c +- add optional "configure" to build zlib as a shared library +- suppress Makefile.qnx, use configure instead +- fixed deflate for 64-bit systems (detected on Cray) +- fixed inflate_blocks for 64-bit systems (detected on Alpha) +- declare Z_DEFLATED in zlib.h (possible parameter for deflateInit2) +- always return Z_BUF_ERROR when deflate() has nothing to do +- deflateInit and inflateInit are now macros to allow version checking +- prefix all global functions and types with z_ with -DZ_PREFIX +- make falloc completely reentrant (inftrees.c) +- fixed very unlikely race condition in ct_static_init +- free in reverse order of allocation to help memory manager +- use zlib-1.0/* instead of zlib/* inside the tar.gz +- make zlib warning-free with "gcc -O3 -Wall -Wwrite-strings -Wpointer-arith + -Wconversion -Wstrict-prototypes -Wmissing-prototypes" +- allow gzread on concatenated .gz files +- deflateEnd now returns Z_DATA_ERROR if it was premature +- deflate is finally (?) fully deterministic (no matches beyond end of input) +- Document Z_SYNC_FLUSH +- add uninstall in Makefile +- Check for __cpluplus in zlib.h +- Better test in ct_align for partial flush +- avoid harmless warnings for Borland C++ +- initialize hash_head in deflate.c +- avoid warning on fdopen (gzio.c) for HP cc -Aa +- include stdlib.h for STDC compilers +- include errno.h for Cray +- ignore error if ranlib doesn't exist +- call ranlib twice for NeXTSTEP +- use exec_prefix instead of prefix for libz.a +- renamed ct_* as _tr_* to avoid conflict with applications +- clear z->msg in inflateInit2 before any error return +- initialize opaque in example.c, gzio.c, deflate.c and inflate.c +- fixed typo in zconf.h (_GNUC__ => __GNUC__) +- check for WIN32 in zconf.h and zutil.c (avoid farmalloc in 32-bit mode) +- fix typo in Make_vms.com (f$trnlnm -> f$getsyi) +- in fcalloc, normalize pointer if size > 65520 bytes +- don't use special fcalloc for 32 bit Borland C++ +- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc... +- use Z_BINARY instead of BINARY +- document that gzclose after gzdopen will close the file +- allow "a" as mode in gzopen. +- fix error checking in gzread +- allow skipping .gz extra-field on pipes +- added reference to Perl interface in README +- put the crc table in FAR data (I dislike more and more the medium model :) +- added get_crc_table +- added a dimension to all arrays (Borland C can't count). +- workaround Borland C bug in declaration of inflate_codes_new & inflate_fast +- guard against multiple inclusion of *.h (for precompiled header on Mac) +- Watcom C pretends to be Microsoft C small model even in 32 bit mode. +- don't use unsized arrays to avoid silly warnings by Visual C++: + warning C4746: 'inflate_mask' : unsized array treated as '__far' + (what's wrong with far data in far model?). +- define enum out of inflate_blocks_state to allow compilation with C++ + +Changes in 0.95 (16 Aug 95) +- fix MSDOS small and medium model (now easier to adapt to any compiler) +- inlined send_bits +- fix the final (:-) bug for deflate with flush (output was correct but + not completely flushed in rare occasions). +- default window size is same for compression and decompression + (it's now sufficient to set MAX_WBITS in zconf.h). +- voidp -> voidpf and voidnp -> voidp (for consistency with other + typedefs and because voidnp was not near in large model). + +Changes in 0.94 (13 Aug 95) +- support MSDOS medium model +- fix deflate with flush (could sometimes generate bad output) +- fix deflateReset (zlib header was incorrectly suppressed) +- added support for VMS +- allow a compression level in gzopen() +- gzflush now calls fflush +- For deflate with flush, flush even if no more input is provided. +- rename libgz.a as libz.a +- avoid complex expression in infcodes.c triggering Turbo C bug +- work around a problem with gcc on Alpha (in INSERT_STRING) +- don't use inline functions (problem with some gcc versions) +- allow renaming of Byte, uInt, etc... with #define. +- avoid warning about (unused) pointer before start of array in deflate.c +- avoid various warnings in gzio.c, example.c, infblock.c, adler32.c, zutil.c +- avoid reserved word 'new' in trees.c + +Changes in 0.93 (25 June 95) +- temporarily disable inline functions +- make deflate deterministic +- give enough lookahead for PARTIAL_FLUSH +- Set binary mode for stdin/stdout in minigzip.c for OS/2 +- don't even use signed char in inflate (not portable enough) +- fix inflate memory leak for segmented architectures + +Changes in 0.92 (3 May 95) +- don't assume that char is signed (problem on SGI) +- Clear bit buffer when starting a stored block +- no memcpy on Pyramid +- suppressed inftest.c +- optimized fill_window, put longest_match inline for gcc +- optimized inflate on stored blocks. +- untabify all sources to simplify patches + +Changes in 0.91 (2 May 95) +- Default MEM_LEVEL is 8 (not 9 for Unix) as documented in zlib.h +- Document the memory requirements in zconf.h +- added "make install" +- fix sync search logic in inflateSync +- deflate(Z_FULL_FLUSH) now works even if output buffer too short +- after inflateSync, don't scare people with just "lo world" +- added support for DJGPP + +Changes in 0.9 (1 May 95) +- don't assume that zalloc clears the allocated memory (the TurboC bug + was Mark's bug after all :) +- let again gzread copy uncompressed data unchanged (was working in 0.71) +- deflate(Z_FULL_FLUSH), inflateReset and inflateSync are now fully implemented +- added a test of inflateSync in example.c +- moved MAX_WBITS to zconf.h because users might want to change that. +- document explicitly that zalloc(64K) on MSDOS must return a normalized + pointer (zero offset) +- added Makefiles for Microsoft C, Turbo C, Borland C++ +- faster crc32() + +Changes in 0.8 (29 April 95) +- added fast inflate (inffast.c) +- deflate(Z_FINISH) now returns Z_STREAM_END when done. Warning: this + is incompatible with previous versions of zlib which returned Z_OK. +- work around a TurboC compiler bug (bad code for b << 0, see infutil.h) + (actually that was not a compiler bug, see 0.81 above) +- gzread no longer reads one extra byte in certain cases +- In gzio destroy(), don't reference a freed structure +- avoid many warnings for MSDOS +- avoid the ERROR symbol which is used by MS Windows + +Changes in 0.71 (14 April 95) +- Fixed more MSDOS compilation problems :( There is still a bug with + TurboC large model. + +Changes in 0.7 (14 April 95) +- Added full inflate support. +- Simplified the crc32() interface. The pre- and post-conditioning + (one's complement) is now done inside crc32(). WARNING: this is + incompatible with previous versions; see zlib.h for the new usage. + +Changes in 0.61 (12 April 95) +- workaround for a bug in TurboC. example and minigzip now work on MSDOS. + +Changes in 0.6 (11 April 95) +- added minigzip.c +- added gzdopen to reopen a file descriptor as gzFile +- added transparent reading of non-gziped files in gzread. +- fixed bug in gzread (don't read crc as data) +- fixed bug in destroy (gzio.c) (don't return Z_STREAM_END for gzclose). +- don't allocate big arrays in the stack (for MSDOS) +- fix some MSDOS compilation problems + +Changes in 0.5: +- do real compression in deflate.c. Z_PARTIAL_FLUSH is supported but + not yet Z_FULL_FLUSH. +- support decompression but only in a single step (forced Z_FINISH) +- added opaque object for zalloc and zfree. +- added deflateReset and inflateReset +- added a variable zlib_version for consistency checking. +- renamed the 'filter' parameter of deflateInit2 as 'strategy'. + Added Z_FILTERED and Z_HUFFMAN_ONLY constants. + +Changes in 0.4: +- avoid "zip" everywhere, use zlib instead of ziplib. +- suppress Z_BLOCK_FLUSH, interpret Z_PARTIAL_FLUSH as block flush + if compression method == 8. +- added adler32 and crc32 +- renamed deflateOptions as deflateInit2, call one or the other but not both +- added the method parameter for deflateInit2. +- added inflateInit2 +- simplied considerably deflateInit and inflateInit by not supporting + user-provided history buffer. This is supported only in deflateInit2 + and inflateInit2. + +Changes in 0.3: +- prefix all macro names with Z_ +- use Z_FINISH instead of deflateEnd to finish compression. +- added Z_HUFFMAN_ONLY +- added gzerror() diff --git a/contrib/zlib/FAQ b/contrib/zlib/FAQ new file mode 100644 index 000000000..99b7cf92e --- /dev/null +++ b/contrib/zlib/FAQ @@ -0,0 +1,368 @@ + + Frequently Asked Questions about zlib + + +If your question is not there, please check the zlib home page +http://zlib.net/ which may have more recent information. +The lastest zlib FAQ is at http://zlib.net/zlib_faq.html + + + 1. Is zlib Y2K-compliant? + + Yes. zlib doesn't handle dates. + + 2. Where can I get a Windows DLL version? + + The zlib sources can be compiled without change to produce a DLL. See the + file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the + precompiled DLL are found in the zlib web site at http://zlib.net/ . + + 3. Where can I get a Visual Basic interface to zlib? + + See + * http://marknelson.us/1997/01/01/zlib-engine/ + * win32/DLL_FAQ.txt in the zlib distribution + + 4. compress() returns Z_BUF_ERROR. + + Make sure that before the call of compress(), the length of the compressed + buffer is equal to the available size of the compressed buffer and not + zero. For Visual Basic, check that this parameter is passed by reference + ("as any"), not by value ("as long"). + + 5. deflate() or inflate() returns Z_BUF_ERROR. + + Before making the call, make sure that avail_in and avail_out are not zero. + When setting the parameter flush equal to Z_FINISH, also make sure that + avail_out is big enough to allow processing all pending input. Note that a + Z_BUF_ERROR is not fatal--another call to deflate() or inflate() can be + made with more input or output space. A Z_BUF_ERROR may in fact be + unavoidable depending on how the functions are used, since it is not + possible to tell whether or not there is more output pending when + strm.avail_out returns with zero. See http://zlib.net/zlib_how.html for a + heavily annotated example. + + 6. Where's the zlib documentation (man pages, etc.)? + + It's in zlib.h . Examples of zlib usage are in the files test/example.c + and test/minigzip.c, with more in examples/ . + + 7. Why don't you use GNU autoconf or libtool or ...? + + Because we would like to keep zlib as a very small and simple package. + zlib is rather portable and doesn't need much configuration. + + 8. I found a bug in zlib. + + Most of the time, such problems are due to an incorrect usage of zlib. + Please try to reproduce the problem with a small program and send the + corresponding source to us at zlib@gzip.org . Do not send multi-megabyte + data files without prior agreement. + + 9. Why do I get "undefined reference to gzputc"? + + If "make test" produces something like + + example.o(.text+0x154): undefined reference to `gzputc' + + check that you don't have old files libz.* in /usr/lib, /usr/local/lib or + /usr/X11R6/lib. Remove any old versions, then do "make install". + +10. I need a Delphi interface to zlib. + + See the contrib/delphi directory in the zlib distribution. + +11. Can zlib handle .zip archives? + + Not by itself, no. See the directory contrib/minizip in the zlib + distribution. + +12. Can zlib handle .Z files? + + No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt + the code of uncompress on your own. + +13. How can I make a Unix shared library? + + By default a shared (and a static) library is built for Unix. So: + + make distclean + ./configure + make + +14. How do I install a shared zlib library on Unix? + + After the above, then: + + make install + + However, many flavors of Unix come with a shared zlib already installed. + Before going to the trouble of compiling a shared version of zlib and + trying to install it, you may want to check if it's already there! If you + can #include , it's there. The -lz option will probably link to + it. You can check the version at the top of zlib.h or with the + ZLIB_VERSION symbol defined in zlib.h . + +15. I have a question about OttoPDF. + + We are not the authors of OttoPDF. The real author is on the OttoPDF web + site: Joel Hainley, jhainley@myndkryme.com. + +16. Can zlib decode Flate data in an Adobe PDF file? + + Yes. See http://www.pdflib.com/ . To modify PDF forms, see + http://sourceforge.net/projects/acroformtool/ . + +17. Why am I getting this "register_frame_info not found" error on Solaris? + + After installing zlib 1.1.4 on Solaris 2.6, running applications using zlib + generates an error such as: + + ld.so.1: rpm: fatal: relocation error: file /usr/local/lib/libz.so: + symbol __register_frame_info: referenced symbol not found + + The symbol __register_frame_info is not part of zlib, it is generated by + the C compiler (cc or gcc). You must recompile applications using zlib + which have this problem. This problem is specific to Solaris. See + http://www.sunfreeware.com for Solaris versions of zlib and applications + using zlib. + +18. Why does gzip give an error on a file I make with compress/deflate? + + The compress and deflate functions produce data in the zlib format, which + is different and incompatible with the gzip format. The gz* functions in + zlib on the other hand use the gzip format. Both the zlib and gzip formats + use the same compressed data format internally, but have different headers + and trailers around the compressed data. + +19. Ok, so why are there two different formats? + + The gzip format was designed to retain the directory information about a + single file, such as the name and last modification date. The zlib format + on the other hand was designed for in-memory and communication channel + applications, and has a much more compact header and trailer and uses a + faster integrity check than gzip. + +20. Well that's nice, but how do I make a gzip file in memory? + + You can request that deflate write the gzip format instead of the zlib + format using deflateInit2(). You can also request that inflate decode the + gzip format using inflateInit2(). Read zlib.h for more details. + +21. Is zlib thread-safe? + + Yes. However any library routines that zlib uses and any application- + provided memory allocation routines must also be thread-safe. zlib's gz* + functions use stdio library routines, and most of zlib's functions use the + library memory allocation routines by default. zlib's *Init* functions + allow for the application to provide custom memory allocation routines. + + Of course, you should only operate on any given zlib or gzip stream from a + single thread at a time. + +22. Can I use zlib in my commercial application? + + Yes. Please read the license in zlib.h. + +23. Is zlib under the GNU license? + + No. Please read the license in zlib.h. + +24. The license says that altered source versions must be "plainly marked". So + what exactly do I need to do to meet that requirement? + + You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In + particular, the final version number needs to be changed to "f", and an + identification string should be appended to ZLIB_VERSION. Version numbers + x.x.x.f are reserved for modifications to zlib by others than the zlib + maintainers. For example, if the version of the base zlib you are altering + is "1.2.3.4", then in zlib.h you should change ZLIB_VERNUM to 0x123f, and + ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also + update the version strings in deflate.c and inftrees.c. + + For altered source distributions, you should also note the origin and + nature of the changes in zlib.h, as well as in ChangeLog and README, along + with the dates of the alterations. The origin should include at least your + name (or your company's name), and an email address to contact for help or + issues with the library. + + Note that distributing a compiled zlib library along with zlib.h and + zconf.h is also a source distribution, and so you should change + ZLIB_VERSION and ZLIB_VERNUM and note the origin and nature of the changes + in zlib.h as you would for a full source distribution. + +25. Will zlib work on a big-endian or little-endian architecture, and can I + exchange compressed data between them? + + Yes and yes. + +26. Will zlib work on a 64-bit machine? + + Yes. It has been tested on 64-bit machines, and has no dependence on any + data types being limited to 32-bits in length. If you have any + difficulties, please provide a complete problem report to zlib@gzip.org + +27. Will zlib decompress data from the PKWare Data Compression Library? + + No. The PKWare DCL uses a completely different compressed data format than + does PKZIP and zlib. However, you can look in zlib's contrib/blast + directory for a possible solution to your problem. + +28. Can I access data randomly in a compressed stream? + + No, not without some preparation. If when compressing you periodically use + Z_FULL_FLUSH, carefully write all the pending data at those points, and + keep an index of those locations, then you can start decompression at those + points. You have to be careful to not use Z_FULL_FLUSH too often, since it + can significantly degrade compression. Alternatively, you can scan a + deflate stream once to generate an index, and then use that index for + random access. See examples/zran.c . + +29. Does zlib work on MVS, OS/390, CICS, etc.? + + It has in the past, but we have not heard of any recent evidence. There + were working ports of zlib 1.1.4 to MVS, but those links no longer work. + If you know of recent, successful applications of zlib on these operating + systems, please let us know. Thanks. + +30. Is there some simpler, easier to read version of inflate I can look at to + understand the deflate format? + + First off, you should read RFC 1951. Second, yes. Look in zlib's + contrib/puff directory. + +31. Does zlib infringe on any patents? + + As far as we know, no. In fact, that was originally the whole point behind + zlib. Look here for some more information: + + http://www.gzip.org/#faq11 + +32. Can zlib work with greater than 4 GB of data? + + Yes. inflate() and deflate() will process any amount of data correctly. + Each call of inflate() or deflate() is limited to input and output chunks + of the maximum value that can be stored in the compiler's "unsigned int" + type, but there is no limit to the number of chunks. Note however that the + strm.total_in and strm_total_out counters may be limited to 4 GB. These + counters are provided as a convenience and are not used internally by + inflate() or deflate(). The application can easily set up its own counters + updated after each call of inflate() or deflate() to count beyond 4 GB. + compress() and uncompress() may be limited to 4 GB, since they operate in a + single call. gzseek() and gztell() may be limited to 4 GB depending on how + zlib is compiled. See the zlibCompileFlags() function in zlib.h. + + The word "may" appears several times above since there is a 4 GB limit only + if the compiler's "long" type is 32 bits. If the compiler's "long" type is + 64 bits, then the limit is 16 exabytes. + +33. Does zlib have any security vulnerabilities? + + The only one that we are aware of is potentially in gzprintf(). If zlib is + compiled to use sprintf() or vsprintf(), then there is no protection + against a buffer overflow of an 8K string space (or other value as set by + gzbuffer()), other than the caller of gzprintf() assuring that the output + will not exceed 8K. On the other hand, if zlib is compiled to use + snprintf() or vsnprintf(), which should normally be the case, then there is + no vulnerability. The ./configure script will display warnings if an + insecure variation of sprintf() will be used by gzprintf(). Also the + zlibCompileFlags() function will return information on what variant of + sprintf() is used by gzprintf(). + + If you don't have snprintf() or vsnprintf() and would like one, you can + find a portable implementation here: + + http://www.ijs.si/software/snprintf/ + + Note that you should be using the most recent version of zlib. Versions + 1.1.3 and before were subject to a double-free vulnerability, and versions + 1.2.1 and 1.2.2 were subject to an access exception when decompressing + invalid compressed data. + +34. Is there a Java version of zlib? + + Probably what you want is to use zlib in Java. zlib is already included + as part of the Java SDK in the java.util.zip package. If you really want + a version of zlib written in the Java language, look on the zlib home + page for links: http://zlib.net/ . + +35. I get this or that compiler or source-code scanner warning when I crank it + up to maximally-pedantic. Can't you guys write proper code? + + Many years ago, we gave up attempting to avoid warnings on every compiler + in the universe. It just got to be a waste of time, and some compilers + were downright silly as well as contradicted each other. So now, we simply + make sure that the code always works. + +36. Valgrind (or some similar memory access checker) says that deflate is + performing a conditional jump that depends on an uninitialized value. + Isn't that a bug? + + No. That is intentional for performance reasons, and the output of deflate + is not affected. This only started showing up recently since zlib 1.2.x + uses malloc() by default for allocations, whereas earlier versions used + calloc(), which zeros out the allocated memory. Even though the code was + correct, versions 1.2.4 and later was changed to not stimulate these + checkers. + +37. Will zlib read the (insert any ancient or arcane format here) compressed + data format? + + Probably not. Look in the comp.compression FAQ for pointers to various + formats and associated software. + +38. How can I encrypt/decrypt zip files with zlib? + + zlib doesn't support encryption. The original PKZIP encryption is very + weak and can be broken with freely available programs. To get strong + encryption, use GnuPG, http://www.gnupg.org/ , which already includes zlib + compression. For PKZIP compatible "encryption", look at + http://www.info-zip.org/ + +39. What's the difference between the "gzip" and "deflate" HTTP 1.1 encodings? + + "gzip" is the gzip format, and "deflate" is the zlib format. They should + probably have called the second one "zlib" instead to avoid confusion with + the raw deflate compressed data format. While the HTTP 1.1 RFC 2616 + correctly points to the zlib specification in RFC 1950 for the "deflate" + transfer encoding, there have been reports of servers and browsers that + incorrectly produce or expect raw deflate data per the deflate + specification in RFC 1951, most notably Microsoft. So even though the + "deflate" transfer encoding using the zlib format would be the more + efficient approach (and in fact exactly what the zlib format was designed + for), using the "gzip" transfer encoding is probably more reliable due to + an unfortunate choice of name on the part of the HTTP 1.1 authors. + + Bottom line: use the gzip format for HTTP 1.1 encoding. + +40. Does zlib support the new "Deflate64" format introduced by PKWare? + + No. PKWare has apparently decided to keep that format proprietary, since + they have not documented it as they have previous compression formats. In + any case, the compression improvements are so modest compared to other more + modern approaches, that it's not worth the effort to implement. + +41. I'm having a problem with the zip functions in zlib, can you help? + + There are no zip functions in zlib. You are probably using minizip by + Giles Vollant, which is found in the contrib directory of zlib. It is not + part of zlib. In fact none of the stuff in contrib is part of zlib. The + files in there are not supported by the zlib authors. You need to contact + the authors of the respective contribution for help. + +42. The match.asm code in contrib is under the GNU General Public License. + Since it's part of zlib, doesn't that mean that all of zlib falls under the + GNU GPL? + + No. The files in contrib are not part of zlib. They were contributed by + other authors and are provided as a convenience to the user within the zlib + distribution. Each item in contrib has its own license. + +43. Is zlib subject to export controls? What is its ECCN? + + zlib is not subject to export controls, and so is classified as EAR99. + +44. Can you please sign these lengthy legal documents and fax them back to us + so that we can use your software in our product? + + No. Go away. Shoo. diff --git a/contrib/zlib/INDEX b/contrib/zlib/INDEX new file mode 100644 index 000000000..2ba064120 --- /dev/null +++ b/contrib/zlib/INDEX @@ -0,0 +1,68 @@ +CMakeLists.txt cmake build file +ChangeLog history of changes +FAQ Frequently Asked Questions about zlib +INDEX this file +Makefile dummy Makefile that tells you to ./configure +Makefile.in template for Unix Makefile +README guess what +configure configure script for Unix +make_vms.com makefile for VMS +test/example.c zlib usages examples for build testing +test/minigzip.c minimal gzip-like functionality for build testing +test/infcover.c inf*.c code coverage for build coverage testing +treebuild.xml XML description of source file dependencies +zconf.h.cmakein zconf.h template for cmake +zconf.h.in zconf.h template for configure +zlib.3 Man page for zlib +zlib.3.pdf Man page in PDF format +zlib.map Linux symbol information +zlib.pc.in Template for pkg-config descriptor +zlib.pc.cmakein zlib.pc template for cmake +zlib2ansi perl script to convert source files for C++ compilation + +amiga/ makefiles for Amiga SAS C +as400/ makefiles for AS/400 +doc/ documentation for formats and algorithms +msdos/ makefiles for MSDOS +nintendods/ makefile for Nintendo DS +old/ makefiles for various architectures and zlib documentation + files that have not yet been updated for zlib 1.2.x +qnx/ makefiles for QNX +watcom/ makefiles for OpenWatcom +win32/ makefiles for Windows + + zlib public header files (required for library use): +zconf.h +zlib.h + + private source files used to build the zlib library: +adler32.c +compress.c +crc32.c +crc32.h +deflate.c +deflate.h +gzclose.c +gzguts.h +gzlib.c +gzread.c +gzwrite.c +infback.c +inffast.c +inffast.h +inffixed.h +inflate.c +inflate.h +inftrees.c +inftrees.h +trees.c +trees.h +uncompr.c +zutil.c +zutil.h + + source files for sample programs +See examples/README.examples + + unsupported contributions by third parties +See contrib/README.contrib diff --git a/contrib/zlib/Makefile.in b/contrib/zlib/Makefile.in new file mode 100644 index 000000000..3413266ae --- /dev/null +++ b/contrib/zlib/Makefile.in @@ -0,0 +1,410 @@ +# Makefile for zlib +# Copyright (C) 1995-2017 Jean-loup Gailly, Mark Adler +# For conditions of distribution and use, see copyright notice in zlib.h + +# To compile and test, type: +# ./configure; make test +# Normally configure builds both a static and a shared library. +# If you want to build just a static library, use: ./configure --static + +# To use the asm code, type: +# cp contrib/asm?86/match.S ./match.S +# make LOC=-DASMV OBJA=match.o + +# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type: +# make install +# To install in $HOME instead of /usr/local, use: +# make install prefix=$HOME + +CC=cc + +CFLAGS=-O +#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#CFLAGS=-g -DZLIB_DEBUG +#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ +# -Wstrict-prototypes -Wmissing-prototypes + +SFLAGS=-O +LDFLAGS= +TEST_LDFLAGS=-L. libz.a +LDSHARED=$(CC) +CPP=$(CC) -E + +STATICLIB=libz.a +SHAREDLIB=libz.so +SHAREDLIBV=libz.so.1.2.11.1 +SHAREDLIBM=libz.so.1 +LIBS=$(STATICLIB) $(SHAREDLIBV) + +AR=ar +ARFLAGS=rc +RANLIB=ranlib +LDCONFIG=ldconfig +LDSHAREDLIBC=-lc +TAR=tar +SHELL=/bin/sh +EXE= + +prefix = /usr/local +exec_prefix = ${prefix} +libdir = ${exec_prefix}/lib +sharedlibdir = ${libdir} +includedir = ${prefix}/include +mandir = ${prefix}/share/man +man3dir = ${mandir}/man3 +pkgconfigdir = ${libdir}/pkgconfig +SRCDIR= +ZINC= +ZINCOUT=-I. + +OBJZ = adler32.o crc32.o deflate.o infback.o inffast.o inflate.o inftrees.o trees.o zutil.o +OBJG = compress.o uncompr.o gzclose.o gzlib.o gzread.o gzwrite.o +OBJC = $(OBJZ) $(OBJG) + +PIC_OBJZ = adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo +PIC_OBJG = compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo +PIC_OBJC = $(PIC_OBJZ) $(PIC_OBJG) + +# to use the asm code: make OBJA=match.o, PIC_OBJA=match.lo +OBJA = +PIC_OBJA = + +OBJS = $(OBJC) $(OBJA) + +PIC_OBJS = $(PIC_OBJC) $(PIC_OBJA) + +all: static shared + +static: example$(EXE) minigzip$(EXE) + +shared: examplesh$(EXE) minigzipsh$(EXE) + +all64: example64$(EXE) minigzip64$(EXE) + +check: test + +test: all teststatic testshared + +teststatic: static + @TMPST=tmpst_$$; \ + if echo hello world | ./minigzip | ./minigzip -d && ./example $$TMPST ; then \ + echo ' *** zlib test OK ***'; \ + else \ + echo ' *** zlib test FAILED ***'; false; \ + fi + @rm -f tmpst_$$ + +testshared: shared + @LD_LIBRARY_PATH=`pwd`:$(LD_LIBRARY_PATH) ; export LD_LIBRARY_PATH; \ + LD_LIBRARYN32_PATH=`pwd`:$(LD_LIBRARYN32_PATH) ; export LD_LIBRARYN32_PATH; \ + DYLD_LIBRARY_PATH=`pwd`:$(DYLD_LIBRARY_PATH) ; export DYLD_LIBRARY_PATH; \ + SHLIB_PATH=`pwd`:$(SHLIB_PATH) ; export SHLIB_PATH; \ + TMPSH=tmpsh_$$; \ + if echo hello world | ./minigzipsh | ./minigzipsh -d && ./examplesh $$TMPSH; then \ + echo ' *** zlib shared test OK ***'; \ + else \ + echo ' *** zlib shared test FAILED ***'; false; \ + fi + @rm -f tmpsh_$$ + +test64: all64 + @TMP64=tmp64_$$; \ + if echo hello world | ./minigzip64 | ./minigzip64 -d && ./example64 $$TMP64; then \ + echo ' *** zlib 64-bit test OK ***'; \ + else \ + echo ' *** zlib 64-bit test FAILED ***'; false; \ + fi + @rm -f tmp64_$$ + +infcover.o: $(SRCDIR)test/infcover.c $(SRCDIR)zlib.h zconf.h + $(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/infcover.c + +infcover: infcover.o libz.a + $(CC) $(CFLAGS) -o $@ infcover.o libz.a + +cover: infcover + rm -f *.gcda + ./infcover + gcov inf*.c + +libz.a: $(OBJS) + $(AR) $(ARFLAGS) $@ $(OBJS) + -@ ($(RANLIB) $@ || true) >/dev/null 2>&1 + +match.o: match.S + $(CPP) match.S > _match.s + $(CC) -c _match.s + mv _match.o match.o + rm -f _match.s + +match.lo: match.S + $(CPP) match.S > _match.s + $(CC) -c -fPIC _match.s + mv _match.o match.lo + rm -f _match.s + +example.o: $(SRCDIR)test/example.c $(SRCDIR)zlib.h zconf.h + $(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/example.c + +minigzip.o: $(SRCDIR)test/minigzip.c $(SRCDIR)zlib.h zconf.h + $(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/minigzip.c + +example64.o: $(SRCDIR)test/example.c $(SRCDIR)zlib.h zconf.h + $(CC) $(CFLAGS) $(ZINCOUT) -D_FILE_OFFSET_BITS=64 -c -o $@ $(SRCDIR)test/example.c + +minigzip64.o: $(SRCDIR)test/minigzip.c $(SRCDIR)zlib.h zconf.h + $(CC) $(CFLAGS) $(ZINCOUT) -D_FILE_OFFSET_BITS=64 -c -o $@ $(SRCDIR)test/minigzip.c + + +adler32.o: $(SRCDIR)adler32.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)adler32.c + +crc32.o: $(SRCDIR)crc32.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)crc32.c + +deflate.o: $(SRCDIR)deflate.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)deflate.c + +infback.o: $(SRCDIR)infback.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)infback.c + +inffast.o: $(SRCDIR)inffast.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)inffast.c + +inflate.o: $(SRCDIR)inflate.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)inflate.c + +inftrees.o: $(SRCDIR)inftrees.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)inftrees.c + +trees.o: $(SRCDIR)trees.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)trees.c + +zutil.o: $(SRCDIR)zutil.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)zutil.c + +compress.o: $(SRCDIR)compress.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)compress.c + +uncompr.o: $(SRCDIR)uncompr.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)uncompr.c + +gzclose.o: $(SRCDIR)gzclose.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzclose.c + +gzlib.o: $(SRCDIR)gzlib.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzlib.c + +gzread.o: $(SRCDIR)gzread.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzread.c + +gzwrite.o: $(SRCDIR)gzwrite.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzwrite.c + + +adler32.lo: $(SRCDIR)adler32.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/adler32.o $(SRCDIR)adler32.c + -@mv objs/adler32.o $@ + +crc32.lo: $(SRCDIR)crc32.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/crc32.o $(SRCDIR)crc32.c + -@mv objs/crc32.o $@ + +deflate.lo: $(SRCDIR)deflate.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/deflate.o $(SRCDIR)deflate.c + -@mv objs/deflate.o $@ + +infback.lo: $(SRCDIR)infback.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/infback.o $(SRCDIR)infback.c + -@mv objs/infback.o $@ + +inffast.lo: $(SRCDIR)inffast.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/inffast.o $(SRCDIR)inffast.c + -@mv objs/inffast.o $@ + +inflate.lo: $(SRCDIR)inflate.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/inflate.o $(SRCDIR)inflate.c + -@mv objs/inflate.o $@ + +inftrees.lo: $(SRCDIR)inftrees.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/inftrees.o $(SRCDIR)inftrees.c + -@mv objs/inftrees.o $@ + +trees.lo: $(SRCDIR)trees.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/trees.o $(SRCDIR)trees.c + -@mv objs/trees.o $@ + +zutil.lo: $(SRCDIR)zutil.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/zutil.o $(SRCDIR)zutil.c + -@mv objs/zutil.o $@ + +compress.lo: $(SRCDIR)compress.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/compress.o $(SRCDIR)compress.c + -@mv objs/compress.o $@ + +uncompr.lo: $(SRCDIR)uncompr.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/uncompr.o $(SRCDIR)uncompr.c + -@mv objs/uncompr.o $@ + +gzclose.lo: $(SRCDIR)gzclose.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzclose.o $(SRCDIR)gzclose.c + -@mv objs/gzclose.o $@ + +gzlib.lo: $(SRCDIR)gzlib.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzlib.o $(SRCDIR)gzlib.c + -@mv objs/gzlib.o $@ + +gzread.lo: $(SRCDIR)gzread.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzread.o $(SRCDIR)gzread.c + -@mv objs/gzread.o $@ + +gzwrite.lo: $(SRCDIR)gzwrite.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzwrite.o $(SRCDIR)gzwrite.c + -@mv objs/gzwrite.o $@ + + +placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a + $(LDSHARED) $(SFLAGS) -o $@ $(PIC_OBJS) $(LDSHAREDLIBC) $(LDFLAGS) + rm -f $(SHAREDLIB) $(SHAREDLIBM) + ln -s $@ $(SHAREDLIB) + ln -s $@ $(SHAREDLIBM) + -@rmdir objs + +example$(EXE): example.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ example.o $(TEST_LDFLAGS) + +minigzip$(EXE): minigzip.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ minigzip.o $(TEST_LDFLAGS) + +examplesh$(EXE): example.o $(SHAREDLIBV) + $(CC) $(CFLAGS) -o $@ example.o -L. $(SHAREDLIBV) + +minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) + $(CC) $(CFLAGS) -o $@ minigzip.o -L. $(SHAREDLIBV) + +example64$(EXE): example64.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ example64.o $(TEST_LDFLAGS) + +minigzip64$(EXE): minigzip64.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ minigzip64.o $(TEST_LDFLAGS) + +install-libs: $(LIBS) + -@if [ ! -d $(DESTDIR)$(exec_prefix) ]; then mkdir -p $(DESTDIR)$(exec_prefix); fi + -@if [ ! -d $(DESTDIR)$(libdir) ]; then mkdir -p $(DESTDIR)$(libdir); fi + -@if [ ! -d $(DESTDIR)$(sharedlibdir) ]; then mkdir -p $(DESTDIR)$(sharedlibdir); fi + -@if [ ! -d $(DESTDIR)$(man3dir) ]; then mkdir -p $(DESTDIR)$(man3dir); fi + -@if [ ! -d $(DESTDIR)$(pkgconfigdir) ]; then mkdir -p $(DESTDIR)$(pkgconfigdir); fi + rm -f $(DESTDIR)$(libdir)/$(STATICLIB) + cp $(STATICLIB) $(DESTDIR)$(libdir) + chmod 644 $(DESTDIR)$(libdir)/$(STATICLIB) + -@($(RANLIB) $(DESTDIR)$(libdir)/libz.a || true) >/dev/null 2>&1 + -@if test -n "$(SHAREDLIBV)"; then \ + rm -f $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV); \ + cp $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir); \ + echo "cp $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)"; \ + chmod 755 $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV); \ + echo "chmod 755 $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV)"; \ + rm -f $(DESTDIR)$(sharedlibdir)/$(SHAREDLIB) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBM); \ + ln -s $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIB); \ + ln -s $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBM); \ + ($(LDCONFIG) || true) >/dev/null 2>&1; \ + fi + rm -f $(DESTDIR)$(man3dir)/zlib.3 + cp $(SRCDIR)zlib.3 $(DESTDIR)$(man3dir) + chmod 644 $(DESTDIR)$(man3dir)/zlib.3 + rm -f $(DESTDIR)$(pkgconfigdir)/zlib.pc + cp zlib.pc $(DESTDIR)$(pkgconfigdir) + chmod 644 $(DESTDIR)$(pkgconfigdir)/zlib.pc +# The ranlib in install is needed on NeXTSTEP which checks file times +# ldconfig is for Linux + +install: install-libs + -@if [ ! -d $(DESTDIR)$(includedir) ]; then mkdir -p $(DESTDIR)$(includedir); fi + rm -f $(DESTDIR)$(includedir)/zlib.h $(DESTDIR)$(includedir)/zconf.h + cp $(SRCDIR)zlib.h zconf.h $(DESTDIR)$(includedir) + chmod 644 $(DESTDIR)$(includedir)/zlib.h $(DESTDIR)$(includedir)/zconf.h + +uninstall: + cd $(DESTDIR)$(includedir) && rm -f zlib.h zconf.h + cd $(DESTDIR)$(libdir) && rm -f libz.a; \ + if test -n "$(SHAREDLIBV)" -a -f $(SHAREDLIBV); then \ + rm -f $(SHAREDLIBV) $(SHAREDLIB) $(SHAREDLIBM); \ + fi + cd $(DESTDIR)$(man3dir) && rm -f zlib.3 + cd $(DESTDIR)$(pkgconfigdir) && rm -f zlib.pc + +docs: zlib.3.pdf + +zlib.3.pdf: $(SRCDIR)zlib.3 + groff -mandoc -f H -T ps $(SRCDIR)zlib.3 | ps2pdf - $@ + +zconf.h.cmakein: $(SRCDIR)zconf.h.in + -@ TEMPFILE=zconfh_$$; \ + echo "/#define ZCONF_H/ a\\\\\n#cmakedefine Z_PREFIX\\\\\n#cmakedefine Z_HAVE_UNISTD_H\n" >> $$TEMPFILE &&\ + sed -f $$TEMPFILE $(SRCDIR)zconf.h.in > $@ &&\ + touch -r $(SRCDIR)zconf.h.in $@ &&\ + rm $$TEMPFILE + +zconf: $(SRCDIR)zconf.h.in + cp -p $(SRCDIR)zconf.h.in zconf.h + +mostlyclean: clean +clean: + rm -f *.o *.lo *~ \ + example$(EXE) minigzip$(EXE) examplesh$(EXE) minigzipsh$(EXE) \ + example64$(EXE) minigzip64$(EXE) \ + infcover \ + libz.* foo.gz so_locations \ + _match.s maketree contrib/infback9/*.o + rm -rf objs + rm -f *.gcda *.gcno *.gcov + rm -f contrib/infback9/*.gcda contrib/infback9/*.gcno contrib/infback9/*.gcov + +maintainer-clean: distclean +distclean: clean zconf zconf.h.cmakein docs + rm -f Makefile zlib.pc configure.log + -@rm -f .DS_Store + @if [ -f Makefile.in ]; then \ + printf 'all:\n\t-@echo "Please use ./configure first. Thank you."\n' > Makefile ; \ + printf '\ndistclean:\n\tmake -f Makefile.in distclean\n' >> Makefile ; \ + touch -r $(SRCDIR)Makefile.in Makefile ; fi + @if [ ! -f zconf.h.in ]; then rm -f zconf.h zconf.h.cmakein ; fi + @if [ ! -f zlib.3 ]; then rm -f zlib.3.pdf ; fi + +tags: + etags $(SRCDIR)*.[ch] + +adler32.o zutil.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h +gzclose.o gzlib.o gzread.o gzwrite.o: $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h +compress.o example.o minigzip.o uncompr.o: $(SRCDIR)zlib.h zconf.h +crc32.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)crc32.h +deflate.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h +infback.o inflate.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h $(SRCDIR)inffixed.h +inffast.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h +inftrees.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h +trees.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h + +adler32.lo zutil.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h +gzclose.lo gzlib.lo gzread.lo gzwrite.lo: $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h +compress.lo example.lo minigzip.lo uncompr.lo: $(SRCDIR)zlib.h zconf.h +crc32.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)crc32.h +deflate.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h +infback.lo inflate.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h $(SRCDIR)inffixed.h +inffast.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h +inftrees.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h +trees.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h diff --git a/contrib/zlib/README b/contrib/zlib/README index 5ca9d127e..41777d06d 100644 --- a/contrib/zlib/README +++ b/contrib/zlib/README @@ -1,6 +1,6 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.2.8 is a general purpose data compression library. All the code is +zlib 1.2.11.1 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and @@ -31,7 +31,7 @@ Mark Nelson wrote an article about zlib for the Jan. 1997 issue of Dr. Dobb's Journal; a copy of the article is available at http://marknelson.us/1997/01/01/zlib-engine/ . -The changes made in version 1.2.8 are documented in the file ChangeLog. +The changes made in version 1.2.11.1 are documented in the file ChangeLog. Unsupported third party contributions are provided in directory contrib/ . @@ -84,7 +84,7 @@ Acknowledgments: Copyright notice: - (C) 1995-2013 Jean-loup Gailly and Mark Adler + (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/contrib/zlib/adler32.c b/contrib/zlib/adler32.c index a868f073d..d0be4380a 100644 --- a/contrib/zlib/adler32.c +++ b/contrib/zlib/adler32.c @@ -1,5 +1,5 @@ /* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2011 Mark Adler + * Copyright (C) 1995-2011, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -7,11 +7,9 @@ #include "zutil.h" -#define local static - local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); -#define BASE 65521 /* largest prime smaller than 65536 */ +#define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -62,10 +60,10 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); #endif /* ========================================================================= */ -uLong ZEXPORT adler32(adler, buf, len) +uLong ZEXPORT adler32_z(adler, buf, len) uLong adler; const Bytef *buf; - uInt len; + z_size_t len; { unsigned long sum2; unsigned n; @@ -132,6 +130,15 @@ uLong ZEXPORT adler32(adler, buf, len) return adler | (sum2 << 16); } +/* ========================================================================= */ +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + /* ========================================================================= */ local uLong adler32_combine_(adler1, adler2, len2) uLong adler1; @@ -156,7 +163,7 @@ local uLong adler32_combine_(adler1, adler2, len2) sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; if (sum1 >= BASE) sum1 -= BASE; if (sum1 >= BASE) sum1 -= BASE; - if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } diff --git a/contrib/zlib/compress.c b/contrib/zlib/compress.c index 6e9762676..e2db404ab 100644 --- a/contrib/zlib/compress.c +++ b/contrib/zlib/compress.c @@ -1,5 +1,5 @@ /* compress.c -- compress a memory buffer - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -28,16 +28,11 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong left; - stream.next_in = (z_const Bytef *)source; - stream.avail_in = (uInt)sourceLen; -#ifdef MAXSEG_64K - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; -#endif - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + left = *destLen; + *destLen = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; @@ -46,15 +41,26 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) err = deflateInit(&stream, level); if (err != Z_OK) return err; - err = deflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&stream); - return err == Z_OK ? Z_BUF_ERROR : err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; - err = deflateEnd(&stream); - return err; + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + sourceLen -= stream.avail_in; + } + err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); + + *destLen = stream.total_out; + deflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : err; } /* =========================================================================== diff --git a/contrib/zlib/configure b/contrib/zlib/configure new file mode 100644 index 000000000..f4127954c --- /dev/null +++ b/contrib/zlib/configure @@ -0,0 +1,921 @@ +#!/bin/sh +# configure script for zlib. +# +# Normally configure builds both a static and a shared library. +# If you want to build just a static library, use: ./configure --static +# +# To impose specific compiler or flags or install directory, use for example: +# prefix=$HOME CC=cc CFLAGS="-O4" ./configure +# or for csh/tcsh users: +# (setenv prefix $HOME; setenv CC cc; setenv CFLAGS "-O4"; ./configure) + +# Incorrect settings of CC or CFLAGS may prevent creating a shared library. +# If you have problems, try without defining CC and CFLAGS before reporting +# an error. + +# start off configure.log +echo -------------------- >> configure.log +echo $0 $* >> configure.log +date >> configure.log + +# get source directory +SRCDIR=`dirname $0` +if test $SRCDIR = "."; then + ZINC="" + ZINCOUT="-I." + SRCDIR="" +else + ZINC='-include zconf.h' + ZINCOUT='-I. -I$(SRCDIR)' + SRCDIR="$SRCDIR/" +fi + +# set command prefix for cross-compilation +if [ -n "${CHOST}" ]; then + uname="`echo "${CHOST}" | sed -e 's/^[^-]*-\([^-]*\)$/\1/' -e 's/^[^-]*-[^-]*-\([^-]*\)$/\1/' -e 's/^[^-]*-[^-]*-\([^-]*\)-.*$/\1/'`" + CROSS_PREFIX="${CHOST}-" +fi + +# destination name for static library +STATICLIB=libz.a + +# extract zlib version numbers from zlib.h +VER=`sed -n -e '/VERSION "/s/.*"\(.*\)".*/\1/p' < ${SRCDIR}zlib.h` +VER3=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\\.[0-9]*\).*/\1/p' < ${SRCDIR}zlib.h` +VER2=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` +VER1=`sed -n -e '/VERSION "/s/.*"\([0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` + +# establish commands for library building +if "${CROSS_PREFIX}ar" --version >/dev/null 2>/dev/null || test $? -lt 126; then + AR=${AR-"${CROSS_PREFIX}ar"} + test -n "${CROSS_PREFIX}" && echo Using ${AR} | tee -a configure.log +else + AR=${AR-"ar"} + test -n "${CROSS_PREFIX}" && echo Using ${AR} | tee -a configure.log +fi +ARFLAGS=${ARFLAGS-"rc"} +if "${CROSS_PREFIX}ranlib" --version >/dev/null 2>/dev/null || test $? -lt 126; then + RANLIB=${RANLIB-"${CROSS_PREFIX}ranlib"} + test -n "${CROSS_PREFIX}" && echo Using ${RANLIB} | tee -a configure.log +else + RANLIB=${RANLIB-"ranlib"} +fi +if "${CROSS_PREFIX}nm" --version >/dev/null 2>/dev/null || test $? -lt 126; then + NM=${NM-"${CROSS_PREFIX}nm"} + test -n "${CROSS_PREFIX}" && echo Using ${NM} | tee -a configure.log +else + NM=${NM-"nm"} +fi + +# set defaults before processing command line options +LDCONFIG=${LDCONFIG-"ldconfig"} +LDSHAREDLIBC="${LDSHAREDLIBC--lc}" +ARCHS= +prefix=${prefix-/usr/local} +exec_prefix=${exec_prefix-'${prefix}'} +libdir=${libdir-'${exec_prefix}/lib'} +sharedlibdir=${sharedlibdir-'${libdir}'} +includedir=${includedir-'${prefix}/include'} +mandir=${mandir-'${prefix}/share/man'} +shared_ext='.so' +shared=1 +solo=0 +cover=0 +zprefix=0 +zconst=0 +build64=0 +gcc=0 +warn=0 +debug=0 +old_cc="$CC" +old_cflags="$CFLAGS" +OBJC='$(OBJZ) $(OBJG)' +PIC_OBJC='$(PIC_OBJZ) $(PIC_OBJG)' + +# leave this script, optionally in a bad way +leave() +{ + if test "$*" != "0"; then + echo "** $0 aborting." | tee -a configure.log + fi + rm -f $test.[co] $test $test$shared_ext $test.gcno ./--version + echo -------------------- >> configure.log + echo >> configure.log + echo >> configure.log + exit $1 +} + +# process command line options +while test $# -ge 1 +do +case "$1" in + -h* | --help) + echo 'usage:' | tee -a configure.log + echo ' configure [--const] [--zprefix] [--prefix=PREFIX] [--eprefix=EXPREFIX]' | tee -a configure.log + echo ' [--static] [--64] [--libdir=LIBDIR] [--sharedlibdir=LIBDIR]' | tee -a configure.log + echo ' [--includedir=INCLUDEDIR] [--archs="-arch i386 -arch x86_64"]' | tee -a configure.log + exit 0 ;; + -p*=* | --prefix=*) prefix=`echo $1 | sed 's/.*=//'`; shift ;; + -e*=* | --eprefix=*) exec_prefix=`echo $1 | sed 's/.*=//'`; shift ;; + -l*=* | --libdir=*) libdir=`echo $1 | sed 's/.*=//'`; shift ;; + --sharedlibdir=*) sharedlibdir=`echo $1 | sed 's/.*=//'`; shift ;; + -i*=* | --includedir=*) includedir=`echo $1 | sed 's/.*=//'`;shift ;; + -u*=* | --uname=*) uname=`echo $1 | sed 's/.*=//'`;shift ;; + -p* | --prefix) prefix="$2"; shift; shift ;; + -e* | --eprefix) exec_prefix="$2"; shift; shift ;; + -l* | --libdir) libdir="$2"; shift; shift ;; + -i* | --includedir) includedir="$2"; shift; shift ;; + -s* | --shared | --enable-shared) shared=1; shift ;; + -t | --static) shared=0; shift ;; + --solo) solo=1; shift ;; + --cover) cover=1; shift ;; + -z* | --zprefix) zprefix=1; shift ;; + -6* | --64) build64=1; shift ;; + -a*=* | --archs=*) ARCHS=`echo $1 | sed 's/.*=//'`; shift ;; + --sysconfdir=*) echo "ignored option: --sysconfdir" | tee -a configure.log; shift ;; + --localstatedir=*) echo "ignored option: --localstatedir" | tee -a configure.log; shift ;; + -c* | --const) zconst=1; shift ;; + -w* | --warn) warn=1; shift ;; + -d* | --debug) debug=1; shift ;; + *) + echo "unknown option: $1" | tee -a configure.log + echo "$0 --help for help" | tee -a configure.log + leave 1;; + esac +done + +# temporary file name +test=ztest$$ + +# put arguments in log, also put test file in log if used in arguments +show() +{ + case "$*" in + *$test.c*) + echo === $test.c === >> configure.log + cat $test.c >> configure.log + echo === >> configure.log;; + esac + echo $* >> configure.log +} + +# check for gcc vs. cc and set compile and link flags based on the system identified by uname +cat > $test.c <&1` in + *gcc*) gcc=1 ;; + *clang*) gcc=1 ;; +esac + +show $cc -c $test.c +if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then + echo ... using gcc >> configure.log + CC="$cc" + CFLAGS="${CFLAGS--O3}" + SFLAGS="${CFLAGS--O3} -fPIC" + if test "$ARCHS"; then + CFLAGS="${CFLAGS} ${ARCHS}" + LDFLAGS="${LDFLAGS} ${ARCHS}" + fi + if test $build64 -eq 1; then + CFLAGS="${CFLAGS} -m64" + SFLAGS="${SFLAGS} -m64" + fi + if test "$warn" -eq 1; then + if test "$zconst" -eq 1; then + CFLAGS="${CFLAGS} -Wall -Wextra -Wcast-qual -pedantic -DZLIB_CONST -fsanitize=address" + else + CFLAGS="${CFLAGS} -Wall -Wextra -pedantic -fsanitize=address" + fi + fi + if test $debug -eq 1; then + CFLAGS="${CFLAGS} -DZLIB_DEBUG" + SFLAGS="${SFLAGS} -DZLIB_DEBUG" + fi + if test -z "$uname"; then + uname=`(uname -s || echo unknown) 2>/dev/null` + fi + case "$uname" in + Linux* | linux* | GNU | GNU/* | solaris*) + LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1,--version-script,${SRCDIR}zlib.map"} ;; + *BSD | *bsd* | DragonFly) + LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1,--version-script,${SRCDIR}zlib.map"} + LDCONFIG="ldconfig -m" ;; + CYGWIN* | Cygwin* | cygwin* | OS/2*) + EXE='.exe' ;; + MINGW* | mingw*) +# temporary bypass + rm -f $test.[co] $test $test$shared_ext + echo "Please use win32/Makefile.gcc instead." | tee -a configure.log + leave 1 + LDSHARED=${LDSHARED-"$cc -shared"} + LDSHAREDLIBC="" + EXE='.exe' ;; + QNX*) # This is for QNX6. I suppose that the QNX rule below is for QNX2,QNX4 + # (alain.bonnefoy@icbt.com) + LDSHARED=${LDSHARED-"$cc -shared -Wl,-hlibz.so.1"} ;; + HP-UX*) + LDSHARED=${LDSHARED-"$cc -shared $SFLAGS"} + case `(uname -m || echo unknown) 2>/dev/null` in + ia64) + shared_ext='.so' + SHAREDLIB='libz.so' ;; + *) + shared_ext='.sl' + SHAREDLIB='libz.sl' ;; + esac ;; + Darwin* | darwin*) + shared_ext='.dylib' + SHAREDLIB=libz$shared_ext + SHAREDLIBV=libz.$VER$shared_ext + SHAREDLIBM=libz.$VER1$shared_ext + LDSHARED=${LDSHARED-"$cc -dynamiclib -install_name $libdir/$SHAREDLIBM -compatibility_version $VER1 -current_version $VER3"} + if libtool -V 2>&1 | grep Apple > /dev/null; then + AR="libtool" + else + AR="/usr/bin/libtool" + fi + ARFLAGS="-o" ;; + *) LDSHARED=${LDSHARED-"$cc -shared"} ;; + esac +else + # find system name and corresponding cc options + CC=${CC-cc} + gcc=0 + echo ... using $CC >> configure.log + if test -z "$uname"; then + uname=`(uname -sr || echo unknown) 2>/dev/null` + fi + case "$uname" in + HP-UX*) SFLAGS=${CFLAGS-"-O +z"} + CFLAGS=${CFLAGS-"-O"} +# LDSHARED=${LDSHARED-"ld -b +vnocompatwarnings"} + LDSHARED=${LDSHARED-"ld -b"} + case `(uname -m || echo unknown) 2>/dev/null` in + ia64) + shared_ext='.so' + SHAREDLIB='libz.so' ;; + *) + shared_ext='.sl' + SHAREDLIB='libz.sl' ;; + esac ;; + IRIX*) SFLAGS=${CFLAGS-"-ansi -O2 -rpath ."} + CFLAGS=${CFLAGS-"-ansi -O2"} + LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so.1"} ;; + OSF1\ V4*) SFLAGS=${CFLAGS-"-O -std1"} + CFLAGS=${CFLAGS-"-O -std1"} + LDFLAGS="${LDFLAGS} -Wl,-rpath,." + LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so -Wl,-msym -Wl,-rpath,$(libdir) -Wl,-set_version,${VER}:1.0"} ;; + OSF1*) SFLAGS=${CFLAGS-"-O -std1"} + CFLAGS=${CFLAGS-"-O -std1"} + LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so.1"} ;; + QNX*) SFLAGS=${CFLAGS-"-4 -O"} + CFLAGS=${CFLAGS-"-4 -O"} + LDSHARED=${LDSHARED-"cc"} + RANLIB=${RANLIB-"true"} + AR="cc" + ARFLAGS="-A" ;; + SCO_SV\ 3.2*) SFLAGS=${CFLAGS-"-O3 -dy -KPIC "} + CFLAGS=${CFLAGS-"-O3"} + LDSHARED=${LDSHARED-"cc -dy -KPIC -G"} ;; + SunOS\ 5* | solaris*) + LDSHARED=${LDSHARED-"cc -G -h libz$shared_ext.$VER1"} + SFLAGS=${CFLAGS-"-fast -KPIC"} + CFLAGS=${CFLAGS-"-fast"} + if test $build64 -eq 1; then + # old versions of SunPRO/Workshop/Studio don't support -m64, + # but newer ones do. Check for it. + flag64=`$CC -flags | egrep -- '^-m64'` + if test x"$flag64" != x"" ; then + CFLAGS="${CFLAGS} -m64" + SFLAGS="${SFLAGS} -m64" + else + case `(uname -m || echo unknown) 2>/dev/null` in + i86*) + SFLAGS="$SFLAGS -xarch=amd64" + CFLAGS="$CFLAGS -xarch=amd64" ;; + *) + SFLAGS="$SFLAGS -xarch=v9" + CFLAGS="$CFLAGS -xarch=v9" ;; + esac + fi + fi + if test -n "$ZINC"; then + ZINC='-I- -I. -I$(SRCDIR)' + fi + ;; + SunOS\ 4*) SFLAGS=${CFLAGS-"-O2 -PIC"} + CFLAGS=${CFLAGS-"-O2"} + LDSHARED=${LDSHARED-"ld"} ;; + SunStudio\ 9*) SFLAGS=${CFLAGS-"-fast -xcode=pic32 -xtarget=ultra3 -xarch=v9b"} + CFLAGS=${CFLAGS-"-fast -xtarget=ultra3 -xarch=v9b"} + LDSHARED=${LDSHARED-"cc -xarch=v9b"} ;; + UNIX_System_V\ 4.2.0) + SFLAGS=${CFLAGS-"-KPIC -O"} + CFLAGS=${CFLAGS-"-O"} + LDSHARED=${LDSHARED-"cc -G"} ;; + UNIX_SV\ 4.2MP) + SFLAGS=${CFLAGS-"-Kconform_pic -O"} + CFLAGS=${CFLAGS-"-O"} + LDSHARED=${LDSHARED-"cc -G"} ;; + OpenUNIX\ 5) + SFLAGS=${CFLAGS-"-KPIC -O"} + CFLAGS=${CFLAGS-"-O"} + LDSHARED=${LDSHARED-"cc -G"} ;; + AIX*) # Courtesy of dbakker@arrayasolutions.com + SFLAGS=${CFLAGS-"-O -qmaxmem=8192"} + CFLAGS=${CFLAGS-"-O -qmaxmem=8192"} + LDSHARED=${LDSHARED-"xlc -G"} ;; + # send working options for other systems to zlib@gzip.org + *) SFLAGS=${CFLAGS-"-O"} + CFLAGS=${CFLAGS-"-O"} + LDSHARED=${LDSHARED-"cc -shared"} ;; + esac +fi + +# destination names for shared library if not defined above +SHAREDLIB=${SHAREDLIB-"libz$shared_ext"} +SHAREDLIBV=${SHAREDLIBV-"libz$shared_ext.$VER"} +SHAREDLIBM=${SHAREDLIBM-"libz$shared_ext.$VER1"} + +echo >> configure.log + +# define functions for testing compiler and library characteristics and logging the results + +cat > $test.c </dev/null; then + try() + { + show $* + test "`( $* ) 2>&1 | tee -a configure.log`" = "" + } + echo - using any output from compiler to indicate an error >> configure.log +else + try() + { + show $* + ( $* ) >> configure.log 2>&1 + ret=$? + if test $ret -ne 0; then + echo "(exit code "$ret")" >> configure.log + fi + return $ret + } +fi + +tryboth() +{ + show $* + got=`( $* ) 2>&1` + ret=$? + printf %s "$got" >> configure.log + if test $ret -ne 0; then + return $ret + fi + test "$got" = "" +} + +cat > $test.c << EOF +int foo() { return 0; } +EOF +echo "Checking for obsessive-compulsive compiler options..." >> configure.log +if try $CC -c $CFLAGS $test.c; then + : +else + echo "Compiler error reporting is too harsh for $0 (perhaps remove -Werror)." | tee -a configure.log + leave 1 +fi + +echo >> configure.log + +# see if shared library build supported +cat > $test.c <> configure.log + show "$NM $test.o | grep _hello" + if test "`$NM $test.o | grep _hello | tee -a configure.log`" = ""; then + CPP="$CPP -DNO_UNDERLINE" + echo Checking for underline in external names... No. | tee -a configure.log + else + echo Checking for underline in external names... Yes. | tee -a configure.log + fi ;; +esac + +echo >> configure.log + +# check for size_t +cat > $test.c < +#include +size_t dummy = 0; +EOF +if try $CC -c $CFLAGS $test.c; then + echo "Checking for size_t... Yes." | tee -a configure.log + need_sizet=0 +else + echo "Checking for size_t... No." | tee -a configure.log + need_sizet=1 +fi + +echo >> configure.log + +# find the size_t integer type, if needed +if test $need_sizet -eq 1; then + cat > $test.c < $test.c < +int main(void) { + if (sizeof(void *) <= sizeof(int)) puts("int"); + else if (sizeof(void *) <= sizeof(long)) puts("long"); + else puts("z_longlong"); + return 0; +} +EOF + else + echo "Checking for long long... No." | tee -a configure.log + cat > $test.c < +int main(void) { + if (sizeof(void *) <= sizeof(int)) puts("int"); + else puts("long"); + return 0; +} +EOF + fi + if try $CC $CFLAGS -o $test $test.c; then + sizet=`./$test` + echo "Checking for a pointer-size integer type..." $sizet"." | tee -a configure.log + else + echo "Failed to find a pointer-size integer type." | tee -a configure.log + leave 1 + fi +fi + +if test $need_sizet -eq 1; then + CFLAGS="${CFLAGS} -DNO_SIZE_T=${sizet}" + SFLAGS="${SFLAGS} -DNO_SIZE_T=${sizet}" +fi + +echo >> configure.log + +# check for large file support, and if none, check for fseeko() +cat > $test.c < +off64_t dummy = 0; +EOF +if try $CC -c $CFLAGS -D_LARGEFILE64_SOURCE=1 $test.c; then + CFLAGS="${CFLAGS} -D_LARGEFILE64_SOURCE=1" + SFLAGS="${SFLAGS} -D_LARGEFILE64_SOURCE=1" + ALL="${ALL} all64" + TEST="${TEST} test64" + echo "Checking for off64_t... Yes." | tee -a configure.log + echo "Checking for fseeko... Yes." | tee -a configure.log +else + echo "Checking for off64_t... No." | tee -a configure.log + echo >> configure.log + cat > $test.c < +int main(void) { + fseeko(NULL, 0, 0); + return 0; +} +EOF + if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for fseeko... Yes." | tee -a configure.log + else + CFLAGS="${CFLAGS} -DNO_FSEEKO" + SFLAGS="${SFLAGS} -DNO_FSEEKO" + echo "Checking for fseeko... No." | tee -a configure.log + fi +fi + +echo >> configure.log + +# check for strerror() for use by gz* functions +cat > $test.c < +#include +int main() { return strlen(strerror(errno)); } +EOF +if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for strerror... Yes." | tee -a configure.log +else + CFLAGS="${CFLAGS} -DNO_STRERROR" + SFLAGS="${SFLAGS} -DNO_STRERROR" + echo "Checking for strerror... No." | tee -a configure.log +fi + +# copy clean zconf.h for subsequent edits +cp -p ${SRCDIR}zconf.h.in zconf.h + +echo >> configure.log + +# check for unistd.h and save result in zconf.h +cat > $test.c < +int main() { return 0; } +EOF +if try $CC -c $CFLAGS $test.c; then + sed < zconf.h "/^#ifdef HAVE_UNISTD_H.* may be/s/def HAVE_UNISTD_H\(.*\) may be/ 1\1 was/" > zconf.temp.h + mv zconf.temp.h zconf.h + echo "Checking for unistd.h... Yes." | tee -a configure.log +else + echo "Checking for unistd.h... No." | tee -a configure.log +fi + +echo >> configure.log + +# check for stdarg.h and save result in zconf.h +cat > $test.c < +int main() { return 0; } +EOF +if try $CC -c $CFLAGS $test.c; then + sed < zconf.h "/^#ifdef HAVE_STDARG_H.* may be/s/def HAVE_STDARG_H\(.*\) may be/ 1\1 was/" > zconf.temp.h + mv zconf.temp.h zconf.h + echo "Checking for stdarg.h... Yes." | tee -a configure.log +else + echo "Checking for stdarg.h... No." | tee -a configure.log +fi + +# if the z_ prefix was requested, save that in zconf.h +if test $zprefix -eq 1; then + sed < zconf.h "/#ifdef Z_PREFIX.* may be/s/def Z_PREFIX\(.*\) may be/ 1\1 was/" > zconf.temp.h + mv zconf.temp.h zconf.h + echo >> configure.log + echo "Using z_ prefix on all symbols." | tee -a configure.log +fi + +# if --solo compilation was requested, save that in zconf.h and remove gz stuff from object lists +if test $solo -eq 1; then + sed '/#define ZCONF_H/a\ +#define Z_SOLO + +' < zconf.h > zconf.temp.h + mv zconf.temp.h zconf.h +OBJC='$(OBJZ)' +PIC_OBJC='$(PIC_OBJZ)' +fi + +# if code coverage testing was requested, use older gcc if defined, e.g. "gcc-4.2" on Mac OS X +if test $cover -eq 1; then + CFLAGS="${CFLAGS} -fprofile-arcs -ftest-coverage" + if test -n "$GCC_CLASSIC"; then + CC=$GCC_CLASSIC + fi +fi + +echo >> configure.log + +# conduct a series of tests to resolve eight possible cases of using "vs" or "s" printf functions +# (using stdarg or not), with or without "n" (proving size of buffer), and with or without a +# return value. The most secure result is vsnprintf() with a return value. snprintf() with a +# return value is secure as well, but then gzprintf() will be limited to 20 arguments. +cat > $test.c < +#include +#include "zconf.h" +int main() +{ +#ifndef STDC + choke me +#endif + return 0; +} +EOF +if try $CC -c $CFLAGS $test.c; then + echo "Checking whether to use vs[n]printf() or s[n]printf()... using vs[n]printf()." | tee -a configure.log + + echo >> configure.log + cat > $test.c < +#include +int mytest(const char *fmt, ...) +{ + char buf[20]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + return 0; +} +int main() +{ + return (mytest("Hello%d\n", 1)); +} +EOF + if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for vsnprintf() in stdio.h... Yes." | tee -a configure.log + + echo >> configure.log + cat >$test.c < +#include +int mytest(const char *fmt, ...) +{ + int n; + char buf[20]; + va_list ap; + va_start(ap, fmt); + n = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + return n; +} +int main() +{ + return (mytest("Hello%d\n", 1)); +} +EOF + + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of vsnprintf()... Yes." | tee -a configure.log + else + CFLAGS="$CFLAGS -DHAS_vsnprintf_void" + SFLAGS="$SFLAGS -DHAS_vsnprintf_void" + echo "Checking for return value of vsnprintf()... No." | tee -a configure.log + echo " WARNING: apparently vsnprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + fi + else + CFLAGS="$CFLAGS -DNO_vsnprintf" + SFLAGS="$SFLAGS -DNO_vsnprintf" + echo "Checking for vsnprintf() in stdio.h... No." | tee -a configure.log + echo " WARNING: vsnprintf() not found, falling back to vsprintf(). zlib" | tee -a configure.log + echo " can build but will be open to possible buffer-overflow security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + + echo >> configure.log + cat >$test.c < +#include +int mytest(const char *fmt, ...) +{ + int n; + char buf[20]; + va_list ap; + va_start(ap, fmt); + n = vsprintf(buf, fmt, ap); + va_end(ap); + return n; +} +int main() +{ + return (mytest("Hello%d\n", 1)); +} +EOF + + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of vsprintf()... Yes." | tee -a configure.log + else + CFLAGS="$CFLAGS -DHAS_vsprintf_void" + SFLAGS="$SFLAGS -DHAS_vsprintf_void" + echo "Checking for return value of vsprintf()... No." | tee -a configure.log + echo " WARNING: apparently vsprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + fi + fi +else + echo "Checking whether to use vs[n]printf() or s[n]printf()... using s[n]printf()." | tee -a configure.log + + echo >> configure.log + cat >$test.c < +int mytest() +{ + char buf[20]; + snprintf(buf, sizeof(buf), "%s", "foo"); + return 0; +} +int main() +{ + return (mytest()); +} +EOF + + if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for snprintf() in stdio.h... Yes." | tee -a configure.log + + echo >> configure.log + cat >$test.c < +int mytest() +{ + char buf[20]; + return snprintf(buf, sizeof(buf), "%s", "foo"); +} +int main() +{ + return (mytest()); +} +EOF + + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of snprintf()... Yes." | tee -a configure.log + else + CFLAGS="$CFLAGS -DHAS_snprintf_void" + SFLAGS="$SFLAGS -DHAS_snprintf_void" + echo "Checking for return value of snprintf()... No." | tee -a configure.log + echo " WARNING: apparently snprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + fi + else + CFLAGS="$CFLAGS -DNO_snprintf" + SFLAGS="$SFLAGS -DNO_snprintf" + echo "Checking for snprintf() in stdio.h... No." | tee -a configure.log + echo " WARNING: snprintf() not found, falling back to sprintf(). zlib" | tee -a configure.log + echo " can build but will be open to possible buffer-overflow security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + + echo >> configure.log + cat >$test.c < +int mytest() +{ + char buf[20]; + return sprintf(buf, "%s", "foo"); +} +int main() +{ + return (mytest()); +} +EOF + + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of sprintf()... Yes." | tee -a configure.log + else + CFLAGS="$CFLAGS -DHAS_sprintf_void" + SFLAGS="$SFLAGS -DHAS_sprintf_void" + echo "Checking for return value of sprintf()... No." | tee -a configure.log + echo " WARNING: apparently sprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + fi + fi +fi + +# see if we can hide zlib internal symbols that are linked between separate source files +if test "$gcc" -eq 1; then + echo >> configure.log + cat > $test.c <> configure.log +echo ALL = $ALL >> configure.log +echo AR = $AR >> configure.log +echo ARFLAGS = $ARFLAGS >> configure.log +echo CC = $CC >> configure.log +echo CFLAGS = $CFLAGS >> configure.log +echo CPP = $CPP >> configure.log +echo EXE = $EXE >> configure.log +echo LDCONFIG = $LDCONFIG >> configure.log +echo LDFLAGS = $LDFLAGS >> configure.log +echo LDSHARED = $LDSHARED >> configure.log +echo LDSHAREDLIBC = $LDSHAREDLIBC >> configure.log +echo OBJC = $OBJC >> configure.log +echo PIC_OBJC = $PIC_OBJC >> configure.log +echo RANLIB = $RANLIB >> configure.log +echo SFLAGS = $SFLAGS >> configure.log +echo SHAREDLIB = $SHAREDLIB >> configure.log +echo SHAREDLIBM = $SHAREDLIBM >> configure.log +echo SHAREDLIBV = $SHAREDLIBV >> configure.log +echo STATICLIB = $STATICLIB >> configure.log +echo TEST = $TEST >> configure.log +echo VER = $VER >> configure.log +echo Z_U4 = $Z_U4 >> configure.log +echo SRCDIR = $SRCDIR >> configure.log +echo exec_prefix = $exec_prefix >> configure.log +echo includedir = $includedir >> configure.log +echo libdir = $libdir >> configure.log +echo mandir = $mandir >> configure.log +echo prefix = $prefix >> configure.log +echo sharedlibdir = $sharedlibdir >> configure.log +echo uname = $uname >> configure.log + +# udpate Makefile with the configure results +sed < ${SRCDIR}Makefile.in " +/^CC *=/s#=.*#=$CC# +/^CFLAGS *=/s#=.*#=$CFLAGS# +/^SFLAGS *=/s#=.*#=$SFLAGS# +/^LDFLAGS *=/s#=.*#=$LDFLAGS# +/^LDSHARED *=/s#=.*#=$LDSHARED# +/^CPP *=/s#=.*#=$CPP# +/^STATICLIB *=/s#=.*#=$STATICLIB# +/^SHAREDLIB *=/s#=.*#=$SHAREDLIB# +/^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV# +/^SHAREDLIBM *=/s#=.*#=$SHAREDLIBM# +/^AR *=/s#=.*#=$AR# +/^ARFLAGS *=/s#=.*#=$ARFLAGS# +/^RANLIB *=/s#=.*#=$RANLIB# +/^LDCONFIG *=/s#=.*#=$LDCONFIG# +/^LDSHAREDLIBC *=/s#=.*#=$LDSHAREDLIBC# +/^EXE *=/s#=.*#=$EXE# +/^SRCDIR *=/s#=.*#=$SRCDIR# +/^ZINC *=/s#=.*#=$ZINC# +/^ZINCOUT *=/s#=.*#=$ZINCOUT# +/^prefix *=/s#=.*#=$prefix# +/^exec_prefix *=/s#=.*#=$exec_prefix# +/^libdir *=/s#=.*#=$libdir# +/^sharedlibdir *=/s#=.*#=$sharedlibdir# +/^includedir *=/s#=.*#=$includedir# +/^mandir *=/s#=.*#=$mandir# +/^OBJC *=/s#=.*#= $OBJC# +/^PIC_OBJC *=/s#=.*#= $PIC_OBJC# +/^all: */s#:.*#: $ALL# +/^test: */s#:.*#: $TEST# +" > Makefile + +# create zlib.pc with the configure results +sed < ${SRCDIR}zlib.pc.in " +/^CC *=/s#=.*#=$CC# +/^CFLAGS *=/s#=.*#=$CFLAGS# +/^CPP *=/s#=.*#=$CPP# +/^LDSHARED *=/s#=.*#=$LDSHARED# +/^STATICLIB *=/s#=.*#=$STATICLIB# +/^SHAREDLIB *=/s#=.*#=$SHAREDLIB# +/^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV# +/^SHAREDLIBM *=/s#=.*#=$SHAREDLIBM# +/^AR *=/s#=.*#=$AR# +/^ARFLAGS *=/s#=.*#=$ARFLAGS# +/^RANLIB *=/s#=.*#=$RANLIB# +/^EXE *=/s#=.*#=$EXE# +/^prefix *=/s#=.*#=$prefix# +/^exec_prefix *=/s#=.*#=$exec_prefix# +/^libdir *=/s#=.*#=$libdir# +/^sharedlibdir *=/s#=.*#=$sharedlibdir# +/^includedir *=/s#=.*#=$includedir# +/^mandir *=/s#=.*#=$mandir# +/^LDFLAGS *=/s#=.*#=$LDFLAGS# +" | sed -e " +s/\@VERSION\@/$VER/g; +" > zlib.pc + +# done +leave 0 diff --git a/contrib/zlib/crc32.c b/contrib/zlib/crc32.c index 979a7190a..e72636a50 100644 --- a/contrib/zlib/crc32.c +++ b/contrib/zlib/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler + * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster @@ -30,17 +30,15 @@ #include "zutil.h" /* for STDC and FAR definitions */ -#define local static - /* Definitions for doing the crc four data bytes at a time. */ #if !defined(NOBYFOUR) && defined(Z_U4) # define BYFOUR #endif #ifdef BYFOUR local unsigned long crc32_little OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); local unsigned long crc32_big OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); # define TBLS 8 #else # define TBLS 1 @@ -201,10 +199,10 @@ const z_crc_t FAR * ZEXPORT get_crc_table() #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) +unsigned long ZEXPORT crc32_z(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - uInt len; + z_size_t len; { if (buf == Z_NULL) return 0UL; @@ -214,7 +212,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) #endif /* DYNAMIC_CRC_TABLE */ #ifdef BYFOUR - if (sizeof(void *) == sizeof(ptrdiff_t)) { + if (sizeof(void *) == sizeof(z_size_t)) { z_crc_t endian; endian = 1; @@ -235,8 +233,29 @@ unsigned long ZEXPORT crc32(crc, buf, len) return crc ^ 0xffffffffUL; } +/* ========================================================================= */ +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ + return crc32_z(crc, buf, len); +} + #ifdef BYFOUR +/* + This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit + integer pointer type. This violates the strict aliasing rule, where a + compiler can assume, for optimization purposes, that two pointers to + fundamentally different types won't ever point to the same memory. This can + manifest as a problem only if one of the pointers is written to. This code + only reads from those pointers. So long as this code remains isolated in + this compilation unit, there won't be a problem. For this reason, this code + should not be copied and pasted into a compilation unit in which other code + writes to the buffer that is passed to these routines. + */ + /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ @@ -247,14 +266,14 @@ unsigned long ZEXPORT crc32(crc, buf, len) local unsigned long crc32_little(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { register z_crc_t c; register const z_crc_t FAR *buf4; c = (z_crc_t)crc; c = ~c; - while (len && ((ptrdiff_t)buf & 3)) { + while (len && ((z_size_t)buf & 3)) { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); len--; } @@ -278,7 +297,7 @@ local unsigned long crc32_little(crc, buf, len) } /* ========================================================================= */ -#define DOBIG4 c ^= *++buf4; \ +#define DOBIG4 c ^= *buf4++; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 @@ -287,20 +306,19 @@ local unsigned long crc32_little(crc, buf, len) local unsigned long crc32_big(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { register z_crc_t c; register const z_crc_t FAR *buf4; c = ZSWAP32((z_crc_t)crc); c = ~c; - while (len && ((ptrdiff_t)buf & 3)) { + while (len && ((z_size_t)buf & 3)) { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); len--; } buf4 = (const z_crc_t FAR *)(const void FAR *)buf; - buf4--; while (len >= 32) { DOBIG32; len -= 32; @@ -309,7 +327,6 @@ local unsigned long crc32_big(crc, buf, len) DOBIG4; len -= 4; } - buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { diff --git a/contrib/zlib/deflate.c b/contrib/zlib/deflate.c index 696957705..568eaddbe 100644 --- a/contrib/zlib/deflate.c +++ b/contrib/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -52,7 +52,7 @@ #include "deflate.h" const char deflate_copyright[] = - " deflate 1.2.8 Copyright 1995-2013 Jean-loup Gailly and Mark Adler "; + " deflate 1.2.11.1 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -73,6 +73,8 @@ typedef enum { typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); local void fill_window OF((deflate_state *s)); local block_state deflate_stored OF((deflate_state *s, int flush)); local block_state deflate_fast OF((deflate_state *s, int flush)); @@ -84,15 +86,16 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); -local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV +# pragma message("Assembler code may have bugs -- use at your own risk") void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); #else local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif @@ -148,21 +151,14 @@ local const config configuration_table[10] = { * meaning. */ -#define EQUAL 0 -/* result of memcmp for equal strings */ - -#ifndef NO_DUMMY_DECL -struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ -#endif - /* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ -#define RANK(f) (((f) << 1) - ((f) > 4 ? 9 : 0)) +#define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) /* =========================================================================== * Update a hash value with the given input byte - * IN assertion: all calls to to UPDATE_HASH are made with consecutive - * input characters, so that a running hash key can be computed from the - * previous key instead of complete recalculation each time. + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. */ #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) @@ -173,9 +169,9 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ * the previous length of the hash chain. * If this file is compiled with -DFASTEST, the compression level is forced * to 1, and no hash chains are maintained. - * IN assertion: all calls to to INSERT_STRING are made with consecutive - * input characters and the first MIN_MATCH bytes of str are valid - * (except for the last MIN_MATCH-1 bytes of the input file). + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). */ #ifdef FASTEST #define INSERT_STRING(s, str, match_head) \ @@ -194,8 +190,42 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ * prev[] will be initialized on the fly. */ #define CLEAR_HASH(s) \ - s->head[s->hash_size-1] = NIL; \ - zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); + do { \ + s->head[s->hash_size-1] = NIL; \ + zmemzero((Bytef *)s->head, \ + (unsigned)(s->hash_size-1)*sizeof(*s->head)); \ + } while (0) + +/* =========================================================================== + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. + */ +local void slide_hash(s) + deflate_state *s; +{ + unsigned n, m; + Posf *p; + uInt wsize = s->w_size; + + n = s->hash_size; + p = &s->head[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + } while (--n); + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } while (--n); +#endif +} /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) @@ -270,7 +300,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, #endif if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || - strategy < 0 || strategy > Z_FIXED) { + strategy < 0 || strategy > Z_FIXED || (windowBits == 8 && wrap != 1)) { return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ @@ -278,14 +308,15 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, if (s == Z_NULL) return Z_MEM_ERROR; strm->state = (struct internal_state FAR *)s; s->strm = strm; + s->status = INIT_STATE; /* to pass state test in deflateReset() */ s->wrap = wrap; s->gzhead = Z_NULL; - s->w_bits = windowBits; + s->w_bits = (uInt)windowBits; s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - s->hash_bits = memLevel + 7; + s->hash_bits = (uInt)memLevel + 7; s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); @@ -319,6 +350,31 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, return deflateReset(strm); } +/* ========================================================================= + * Check for a valid deflate stream state. Return 0 if ok, 1 if not. + */ +local int deflateStateCheck (strm) + z_streamp strm; +{ + deflate_state *s; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + s = strm->state; + if (s == Z_NULL || s->strm != strm || (s->status != INIT_STATE && +#ifdef GZIP + s->status != GZIP_STATE && +#endif + s->status != EXTRA_STATE && + s->status != NAME_STATE && + s->status != COMMENT_STATE && + s->status != HCRC_STATE && + s->status != BUSY_STATE && + s->status != FINISH_STATE)) + return 1; + return 0; +} + /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; @@ -331,7 +387,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) unsigned avail; z_const unsigned char *next; - if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL) + if (deflateStateCheck(strm) || dictionary == Z_NULL) return Z_STREAM_ERROR; s = strm->state; wrap = s->wrap; @@ -388,14 +444,35 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) return Z_OK; } +/* ========================================================================= */ +int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; + uInt *dictLength; +{ + deflate_state *s; + uInt len; + + if (deflateStateCheck(strm)) + return Z_STREAM_ERROR; + s = strm->state; + len = s->strstart + s->lookahead; + if (len > s->w_size) + len = s->w_size; + if (dictionary != Z_NULL && len) + zmemcpy(dictionary, s->window + s->strstart + s->lookahead - len, len); + if (dictLength != Z_NULL) + *dictLength = len; + return Z_OK; +} + /* ========================================================================= */ int ZEXPORT deflateResetKeep (strm) z_streamp strm; { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) { + if (deflateStateCheck(strm)) { return Z_STREAM_ERROR; } @@ -410,13 +487,17 @@ int ZEXPORT deflateResetKeep (strm) if (s->wrap < 0) { s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ } - s->status = s->wrap ? INIT_STATE : BUSY_STATE; + s->status = +#ifdef GZIP + s->wrap == 2 ? GZIP_STATE : +#endif + s->wrap ? INIT_STATE : BUSY_STATE; strm->adler = #ifdef GZIP s->wrap == 2 ? crc32(0L, Z_NULL, 0) : #endif adler32(0L, Z_NULL, 0); - s->last_flush = Z_NO_FLUSH; + s->last_flush = -2; _tr_init(s); @@ -440,8 +521,8 @@ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - if (strm->state->wrap != 2) return Z_STREAM_ERROR; + if (deflateStateCheck(strm) || strm->state->wrap != 2) + return Z_STREAM_ERROR; strm->state->gzhead = head; return Z_OK; } @@ -452,7 +533,7 @@ int ZEXPORT deflatePending (strm, pending, bits) int *bits; z_streamp strm; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; if (pending != Z_NULL) *pending = strm->state->pending; if (bits != Z_NULL) @@ -469,7 +550,7 @@ int ZEXPORT deflatePrime (strm, bits, value) deflate_state *s; int put; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; @@ -494,9 +575,8 @@ int ZEXPORT deflateParams(strm, level, strategy) { deflate_state *s; compress_func func; - int err = Z_OK; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; #ifdef FASTEST @@ -510,13 +590,22 @@ int ZEXPORT deflateParams(strm, level, strategy) func = configuration_table[s->level].func; if ((strategy != s->strategy || func != configuration_table[level].func) && - strm->total_in != 0) { + s->last_flush != -2) { /* Flush the last buffer: */ - err = deflate(strm, Z_BLOCK); - if (err == Z_BUF_ERROR && s->pending == 0) - err = Z_OK; + int err = deflate(strm, Z_BLOCK); + if (err == Z_STREAM_ERROR) + return err; + if (strm->avail_in || (s->strstart - s->block_start) + s->lookahead) + return Z_BUF_ERROR; } if (s->level != level) { + if (s->level == 0 && s->matches != 0) { + if (s->matches == 1) + slide_hash(s); + else + CLEAR_HASH(s); + s->matches = 0; + } s->level = level; s->max_lazy_match = configuration_table[level].max_lazy; s->good_match = configuration_table[level].good_length; @@ -524,7 +613,7 @@ int ZEXPORT deflateParams(strm, level, strategy) s->max_chain_length = configuration_table[level].max_chain; } s->strategy = strategy; - return err; + return Z_OK; } /* ========================================================================= */ @@ -537,12 +626,12 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; - s->good_match = good_length; - s->max_lazy_match = max_lazy; + s->good_match = (uInt)good_length; + s->max_lazy_match = (uInt)max_lazy; s->nice_match = nice_length; - s->max_chain_length = max_chain; + s->max_chain_length = (uInt)max_chain; return Z_OK; } @@ -569,14 +658,13 @@ uLong ZEXPORT deflateBound(strm, sourceLen) { deflate_state *s; uLong complen, wraplen; - Bytef *str; /* conservative upper bound for compressed data */ complen = sourceLen + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; /* if can't get parameters, return conservative bound plus zlib wrapper */ - if (strm == Z_NULL || strm->state == Z_NULL) + if (deflateStateCheck(strm)) return complen + 6; /* compute wrapper length */ @@ -588,9 +676,11 @@ uLong ZEXPORT deflateBound(strm, sourceLen) case 1: /* zlib wrapper */ wraplen = 6 + (s->strstart ? 4 : 0); break; +#ifdef GZIP case 2: /* gzip wrapper */ wraplen = 18; if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + Bytef *str; if (s->gzhead->extra != Z_NULL) wraplen += 2 + s->gzhead->extra_len; str = s->gzhead->name; @@ -607,6 +697,7 @@ uLong ZEXPORT deflateBound(strm, sourceLen) wraplen += 2; } break; +#endif default: /* for compiler happiness */ wraplen = 6; } @@ -634,10 +725,10 @@ local void putShortMSB (s, b) } /* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). + * Flush as much pending output as possible. All deflate() output, except for + * some deflate_stored() output, goes through this function so some + * applications may wish to modify it to avoid allocating a large + * strm->next_out buffer and copying into it. (See also read_buf()). */ local void flush_pending(strm) z_streamp strm; @@ -654,13 +745,23 @@ local void flush_pending(strm) strm->next_out += len; s->pending_out += len; strm->total_out += len; - strm->avail_out -= len; - s->pending -= len; + strm->avail_out -= len; + s->pending -= len; if (s->pending == 0) { s->pending_out = s->pending_buf; } } +/* =========================================================================== + * Update the header CRC with the bytes s->pending_buf[beg..s->pending - 1]. + */ +#define HCRC_UPDATE(beg) \ + do { \ + if (s->gzhead->hcrc && s->pending > (beg)) \ + strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ + } while (0) + /* ========================================================================= */ int ZEXPORT deflate (strm, flush) z_streamp strm; @@ -669,203 +770,21 @@ int ZEXPORT deflate (strm, flush) int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_BLOCK || flush < 0) { + if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; if (strm->next_out == Z_NULL || - (strm->next_in == Z_NULL && strm->avail_in != 0) || + (strm->avail_in != 0 && strm->next_in == Z_NULL) || (s->status == FINISH_STATE && flush != Z_FINISH)) { ERR_RETURN(strm, Z_STREAM_ERROR); } if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); - s->strm = strm; /* just in case */ old_flush = s->last_flush; s->last_flush = flush; - /* Write the header */ - if (s->status == INIT_STATE) { -#ifdef GZIP - if (s->wrap == 2) { - strm->adler = crc32(0L, Z_NULL, 0); - put_byte(s, 31); - put_byte(s, 139); - put_byte(s, 8); - if (s->gzhead == Z_NULL) { - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, OS_CODE); - s->status = BUSY_STATE; - } - else { - put_byte(s, (s->gzhead->text ? 1 : 0) + - (s->gzhead->hcrc ? 2 : 0) + - (s->gzhead->extra == Z_NULL ? 0 : 4) + - (s->gzhead->name == Z_NULL ? 0 : 8) + - (s->gzhead->comment == Z_NULL ? 0 : 16) - ); - put_byte(s, (Byte)(s->gzhead->time & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, s->gzhead->os & 0xff); - if (s->gzhead->extra != Z_NULL) { - put_byte(s, s->gzhead->extra_len & 0xff); - put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); - } - if (s->gzhead->hcrc) - strm->adler = crc32(strm->adler, s->pending_buf, - s->pending); - s->gzindex = 0; - s->status = EXTRA_STATE; - } - } - else -#endif - { - uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; - uInt level_flags; - - if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) - level_flags = 0; - else if (s->level < 6) - level_flags = 1; - else if (s->level == 6) - level_flags = 2; - else - level_flags = 3; - header |= (level_flags << 6); - if (s->strstart != 0) header |= PRESET_DICT; - header += 31 - (header % 31); - - s->status = BUSY_STATE; - putShortMSB(s, header); - - /* Save the adler32 of the preset dictionary: */ - if (s->strstart != 0) { - putShortMSB(s, (uInt)(strm->adler >> 16)); - putShortMSB(s, (uInt)(strm->adler & 0xffff)); - } - strm->adler = adler32(0L, Z_NULL, 0); - } - } -#ifdef GZIP - if (s->status == EXTRA_STATE) { - if (s->gzhead->extra != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - - while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) - break; - } - put_byte(s, s->gzhead->extra[s->gzindex]); - s->gzindex++; - } - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (s->gzindex == s->gzhead->extra_len) { - s->gzindex = 0; - s->status = NAME_STATE; - } - } - else - s->status = NAME_STATE; - } - if (s->status == NAME_STATE) { - if (s->gzhead->name != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - int val; - - do { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; - } - } - val = s->gzhead->name[s->gzindex++]; - put_byte(s, val); - } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) { - s->gzindex = 0; - s->status = COMMENT_STATE; - } - } - else - s->status = COMMENT_STATE; - } - if (s->status == COMMENT_STATE) { - if (s->gzhead->comment != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - int val; - - do { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; - } - } - val = s->gzhead->comment[s->gzindex++]; - put_byte(s, val); - } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) - s->status = HCRC_STATE; - } - else - s->status = HCRC_STATE; - } - if (s->status == HCRC_STATE) { - if (s->gzhead->hcrc) { - if (s->pending + 2 > s->pending_buf_size) - flush_pending(strm); - if (s->pending + 2 <= s->pending_buf_size) { - put_byte(s, (Byte)(strm->adler & 0xff)); - put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); - strm->adler = crc32(0L, Z_NULL, 0); - s->status = BUSY_STATE; - } - } - else - s->status = BUSY_STATE; - } -#endif - /* Flush as much pending output as possible */ if (s->pending != 0) { flush_pending(strm); @@ -894,15 +813,197 @@ int ZEXPORT deflate (strm, flush) ERR_RETURN(strm, Z_BUF_ERROR); } + /* Write the header */ + if (s->status == INIT_STATE) { + /* zlib header */ + uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; + uInt level_flags; + + if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) + level_flags = 0; + else if (s->level < 6) + level_flags = 1; + else if (s->level == 6) + level_flags = 2; + else + level_flags = 3; + header |= (level_flags << 6); + if (s->strstart != 0) header |= PRESET_DICT; + header += 31 - (header % 31); + + putShortMSB(s, header); + + /* Save the adler32 of the preset dictionary: */ + if (s->strstart != 0) { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + strm->adler = adler32(0L, Z_NULL, 0); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ + strm->adler = crc32(0L, Z_NULL, 0); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); + if (s->gzhead == Z_NULL) { + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, OS_CODE); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + else { + put_byte(s, (s->gzhead->text ? 1 : 0) + + (s->gzhead->hcrc ? 2 : 0) + + (s->gzhead->extra == Z_NULL ? 0 : 4) + + (s->gzhead->name == Z_NULL ? 0 : 8) + + (s->gzhead->comment == Z_NULL ? 0 : 16) + ); + put_byte(s, (Byte)(s->gzhead->time & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, s->gzhead->os & 0xff); + if (s->gzhead->extra != Z_NULL) { + put_byte(s, s->gzhead->extra_len & 0xff); + put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); + } + if (s->gzhead->hcrc) + strm->adler = crc32(strm->adler, s->pending_buf, + s->pending); + s->gzindex = 0; + s->status = EXTRA_STATE; + } + } + if (s->status == EXTRA_STATE) { + if (s->gzhead->extra != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + while (s->pending + left > s->pending_buf_size) { + uInt copy = s->pending_buf_size - s->pending; + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, copy); + s->pending = s->pending_buf_size; + HCRC_UPDATE(beg); + s->gzindex += copy; + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + left -= copy; + } + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, left); + s->pending += left; + HCRC_UPDATE(beg); + s->gzindex = 0; + } + s->status = NAME_STATE; + } + if (s->status == NAME_STATE) { + if (s->gzhead->name != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + int val; + do { + if (s->pending == s->pending_buf_size) { + HCRC_UPDATE(beg); + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + } + val = s->gzhead->name[s->gzindex++]; + put_byte(s, val); + } while (val != 0); + HCRC_UPDATE(beg); + s->gzindex = 0; + } + s->status = COMMENT_STATE; + } + if (s->status == COMMENT_STATE) { + if (s->gzhead->comment != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + int val; + do { + if (s->pending == s->pending_buf_size) { + HCRC_UPDATE(beg); + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + } + val = s->gzhead->comment[s->gzindex++]; + put_byte(s, val); + } while (val != 0); + HCRC_UPDATE(beg); + } + s->status = HCRC_STATE; + } + if (s->status == HCRC_STATE) { + if (s->gzhead->hcrc) { + if (s->pending + 2 > s->pending_buf_size) { + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + strm->adler = crc32(0L, Z_NULL, 0); + } + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#endif + /* Start a new block or continue the current one. */ if (strm->avail_in != 0 || s->lookahead != 0 || (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : - (s->strategy == Z_RLE ? deflate_rle(s, flush) : - (*(configuration_table[s->level].func))(s, flush)); + bstate = s->level == 0 ? deflate_stored(s, flush) : + s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -944,7 +1045,6 @@ int ZEXPORT deflate (strm, flush) } } } - Assert(strm->avail_out > 0, "bug2"); if (flush != Z_FINISH) return Z_OK; if (s->wrap <= 0) return Z_STREAM_END; @@ -981,18 +1081,9 @@ int ZEXPORT deflateEnd (strm) { int status; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; status = strm->state->status; - if (status != INIT_STATE && - status != EXTRA_STATE && - status != NAME_STATE && - status != COMMENT_STATE && - status != HCRC_STATE && - status != BUSY_STATE && - status != FINISH_STATE) { - return Z_STREAM_ERROR; - } /* Deallocate in reverse order of allocations: */ TRY_FREE(strm, strm->state->pending_buf); @@ -1023,7 +1114,7 @@ int ZEXPORT deflateCopy (dest, source) ushf *overlay; - if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) { + if (deflateStateCheck(source) || dest == Z_NULL) { return Z_STREAM_ERROR; } @@ -1073,7 +1164,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -local int read_buf(strm, buf, size) +local unsigned read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -1097,7 +1188,7 @@ local int read_buf(strm, buf, size) strm->next_in += len; strm->total_in += len; - return (int)len; + return len; } /* =========================================================================== @@ -1151,9 +1242,9 @@ local uInt longest_match(s, cur_match) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ + register Bytef *match; /* matched string */ register int len; /* length of current match */ - int best_len = s->prev_length; /* best match length so far */ + int best_len = (int)s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; @@ -1188,7 +1279,7 @@ local uInt longest_match(s, cur_match) /* Do not look for matches beyond the end of the input. This is necessary * to make deflate deterministic. */ - if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; + if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); @@ -1349,7 +1440,11 @@ local uInt longest_match(s, cur_match) #endif /* FASTEST */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG + +#define EQUAL 0 +/* result of memcmp for equal strings */ + /* =========================================================================== * Check that the match at match_start is indeed a match. */ @@ -1375,7 +1470,7 @@ local void check_match(s, start, match, length) } #else # define check_match(s, start, match, length) -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* =========================================================================== * Fill the window when the lookahead becomes insufficient. @@ -1390,8 +1485,7 @@ local void check_match(s, start, match, length) local void fill_window(s) deflate_state *s; { - register unsigned n, m; - register Posf *p; + unsigned n; unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; @@ -1418,35 +1512,13 @@ local void fill_window(s) */ if (s->strstart >= wsize+MAX_DIST(s)) { - zmemcpy(s->window, s->window+wsize, (unsigned)wsize); + zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ s->block_start -= (long) wsize; - - /* Slide the hash table (could be avoided with 32 bit values - at the expense of memory usage). We slide even when level == 0 - to keep the hash table consistent if we switch back to level > 0 - later. (Using level 0 permanently is not an optimal usage of - zlib, so we don't care about this pathological case.) - */ - n = s->hash_size; - p = &s->head[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - } while (--n); - - n = wsize; -#ifndef FASTEST - p = &s->prev[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - /* If n is not on any hash chain, prev[n] is garbage but - * its value will never be used. - */ - } while (--n); -#endif + if (s->insert > s->strstart) + s->insert = s->strstart; + slide_hash(s); more += wsize; } if (s->strm->avail_in == 0) break; @@ -1552,70 +1624,205 @@ local void fill_window(s) if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } +/* Maximum stored block length in deflate format (not including header). */ +#define MAX_STORED 65535 + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + /* =========================================================================== * Copy without compression as much as possible from the input stream, return * the current block state. - * This function does not insert new strings in the dictionary since - * uncompressible data is probably not useful. This function is used - * only for the level=0 compression option. - * NOTE: this function should be optimized to avoid extra copying from - * window to pending_buf. + * + * In case deflateParams() is used to later switch to a non-zero compression + * level, s->matches (otherwise unused when storing) keeps track of the number + * of hash table slides to perform. If s->matches is 1, then one hash table + * slide will be done when switching. If s->matches is 2, the maximum value + * allowed here, then the hash table will be cleared, since two or more slides + * is the same as a clear. + * + * deflate_stored() is written to minimize the number of times an input byte is + * copied. It is most efficient with large input and output buffers, which + * maximizes the opportunites to have a single copy from next_in to next_out. */ local block_state deflate_stored(s, flush) deflate_state *s; int flush; { - /* Stored blocks are limited to 0xffff bytes, pending_buf is limited - * to pending_buf_size, and each stored block has a 5 byte header: + /* Smallest worthy block size when not flushing or finishing. By default + * this is 32K. This can be as small as 507 bytes for memLevel == 1. For + * large input and output buffers, the stored block size will be larger. */ - ulg max_block_size = 0xffff; - ulg max_start; + unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); - if (max_block_size > s->pending_buf_size - 5) { - max_block_size = s->pending_buf_size - 5; - } - - /* Copy as much as possible from input to output: */ - for (;;) { - /* Fill the window as much as possible: */ - if (s->lookahead <= 1) { - - Assert(s->strstart < s->w_size+MAX_DIST(s) || - s->block_start >= (long)s->w_size, "slide too late"); - - fill_window(s); - if (s->lookahead == 0 && flush == Z_NO_FLUSH) return need_more; - - if (s->lookahead == 0) break; /* flush the current block */ - } - Assert(s->block_start >= 0L, "block gone"); - - s->strstart += s->lookahead; - s->lookahead = 0; - - /* Emit a stored block if pending_buf will be full: */ - max_start = s->block_start + max_block_size; - if (s->strstart == 0 || (ulg)s->strstart >= max_start) { - /* strstart == 0 is possible when wraparound on 16-bit machine */ - s->lookahead = (uInt)(s->strstart - max_start); - s->strstart = (uInt)max_start; - FLUSH_BLOCK(s, 0); - } - /* Flush if we may have to slide, otherwise block_start may become - * negative and the data will be gone: + /* Copy as many min_block or larger stored blocks directly to next_out as + * possible. If flushing, copy the remaining available input to next_out as + * stored blocks, if there is enough space. + */ + unsigned len, left, have, last = 0; + unsigned used = s->strm->avail_in; + do { + /* Set len to the maximum size block that we can copy directly with the + * available input data and output space. Set left to how much of that + * would be copied from what's left in the window. */ - if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) { - FLUSH_BLOCK(s, 0); + len = MAX_STORED; /* maximum deflate stored block length */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + if (s->strm->avail_out < have) /* need room for header */ + break; + /* maximum stored block length that will fit in avail_out: */ + have = s->strm->avail_out - have; + left = s->strstart - s->block_start; /* bytes left in window */ + if (len > (ulg)left + s->strm->avail_in) + len = left + s->strm->avail_in; /* limit len to the input */ + if (len > have) + len = have; /* limit len to the output */ + + /* If the stored block would be less than min_block in length, or if + * unable to copy all of the available input when flushing, then try + * copying to the window and the pending buffer instead. Also don't + * write an empty block when flushing -- deflate() does that. + */ + if (len < min_block && ((len == 0 && flush != Z_FINISH) || + flush == Z_NO_FLUSH || + len != left + s->strm->avail_in)) + break; + + /* Make a dummy stored block in pending to get the header bytes, + * including any pending bits. This also updates the debugging counts. + */ + last = flush == Z_FINISH && len == left + s->strm->avail_in ? 1 : 0; + _tr_stored_block(s, (char *)0, 0L, last); + + /* Replace the lengths in the dummy stored block with len. */ + s->pending_buf[s->pending - 4] = len; + s->pending_buf[s->pending - 3] = len >> 8; + s->pending_buf[s->pending - 2] = ~len; + s->pending_buf[s->pending - 1] = ~len >> 8; + + /* Write the stored block header bytes. */ + flush_pending(s->strm); + +#ifdef ZLIB_DEBUG + /* Update debugging counts for the data about to be copied. */ + s->compressed_len += len << 3; + s->bits_sent += len << 3; +#endif + + /* Copy uncompressed bytes from the window to next_out. */ + if (left) { + if (left > len) + left = len; + zmemcpy(s->strm->next_out, s->window + s->block_start, left); + s->strm->next_out += left; + s->strm->avail_out -= left; + s->strm->total_out += left; + s->block_start += left; + len -= left; } + + /* Copy uncompressed bytes directly from next_in to next_out, updating + * the check value. + */ + if (len) { + read_buf(s->strm, s->strm->next_out, len); + s->strm->next_out += len; + s->strm->avail_out -= len; + s->strm->total_out += len; + } + } while (last == 0); + + /* Update the sliding window with the last s->w_size bytes of the copied + * data, or append all of the copied data to the existing window if less + * than s->w_size bytes were copied. Also update the number of bytes to + * insert in the hash tables, in the event that deflateParams() switches to + * a non-zero compression level. + */ + used -= s->strm->avail_in; /* number of input bytes directly copied */ + if (used) { + /* If any input was used, then no unused input remains in the window, + * therefore s->block_start == s->strstart. + */ + if (used >= s->w_size) { /* supplant the previous history */ + s->matches = 2; /* clear hash */ + zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size); + s->strstart = s->w_size; + s->insert = s->strstart; + } + else { + if (s->window_size - s->strstart <= used) { + /* Slide the window down. */ + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + if (s->insert > s->strstart) + s->insert = s->strstart; + } + zmemcpy(s->window + s->strstart, s->strm->next_in - used, used); + s->strstart += used; + s->insert += MIN(used, s->w_size - s->insert); + } + s->block_start = s->strstart; } - s->insert = 0; - if (flush == Z_FINISH) { - FLUSH_BLOCK(s, 1); + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* If the last block was written to next_out, then done. */ + if (last) return finish_done; + + /* If flushing and all input has been consumed, then done. */ + if (flush != Z_NO_FLUSH && flush != Z_FINISH && + s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + return block_done; + + /* Fill the window with any remaining input. */ + have = s->window_size - s->strstart; + if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + /* Slide the window down. */ + s->block_start -= s->w_size; + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + have += s->w_size; /* more space now */ + if (s->insert > s->strstart) + s->insert = s->strstart; } - if ((long)s->strstart > s->block_start) - FLUSH_BLOCK(s, 0); - return block_done; + if (have > s->strm->avail_in) + have = s->strm->avail_in; + if (have) { + read_buf(s->strm, s->window + s->strstart, have); + s->strstart += have; + s->insert += MIN(have, s->w_size - s->insert); + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* There was not enough avail_out to write a complete worthy or flushed + * stored block to next_out. Write a stored block to pending instead, if we + * have enough input for a worthy block, or if flushing and there is enough + * room for the remaining input as a stored block in the pending buffer. + */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + /* maximum stored block length that will fit in pending: */ + have = MIN(s->pending_buf_size - have, MAX_STORED); + min_block = MIN(have, s->w_size); + left = s->strstart - s->block_start; + if (left >= min_block || + ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && + s->strm->avail_in == 0 && left <= have)) { + len = MIN(left, have); + last = flush == Z_FINISH && s->strm->avail_in == 0 && + len == left ? 1 : 0; + _tr_stored_block(s, (charf *)s->window + s->block_start, len, last); + s->block_start += len; + flush_pending(s->strm); + } + + /* We've done all we can with the available input and output. */ + return last ? finish_started : need_more; } /* =========================================================================== @@ -1892,7 +2099,7 @@ local block_state deflate_rle(s, flush) prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && scan < strend); - s->match_length = MAX_MATCH - (int)(strend - scan); + s->match_length = MAX_MATCH - (uInt)(strend - scan); if (s->match_length > s->lookahead) s->match_length = s->lookahead; } diff --git a/contrib/zlib/deflate.h b/contrib/zlib/deflate.h index ce0299edd..23ecdd312 100644 --- a/contrib/zlib/deflate.h +++ b/contrib/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2012 Jean-loup Gailly + * Copyright (C) 1995-2016 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -51,13 +51,16 @@ #define Buf_size 16 /* size of bit buffer in bi_buf */ -#define INIT_STATE 42 -#define EXTRA_STATE 69 -#define NAME_STATE 73 -#define COMMENT_STATE 91 -#define HCRC_STATE 103 -#define BUSY_STATE 113 -#define FINISH_STATE 666 +#define INIT_STATE 42 /* zlib header -> BUSY_STATE */ +#ifdef GZIP +# define GZIP_STATE 57 /* gzip header -> BUSY_STATE | EXTRA_STATE */ +#endif +#define EXTRA_STATE 69 /* gzip extra block -> NAME_STATE */ +#define NAME_STATE 73 /* gzip file name -> COMMENT_STATE */ +#define COMMENT_STATE 91 /* gzip comment -> HCRC_STATE */ +#define HCRC_STATE 103 /* gzip header CRC -> BUSY_STATE */ +#define BUSY_STATE 113 /* deflate -> FINISH_STATE */ +#define FINISH_STATE 666 /* stream complete */ /* Stream status */ @@ -83,7 +86,7 @@ typedef struct static_tree_desc_s static_tree_desc; typedef struct tree_desc_s { ct_data *dyn_tree; /* the dynamic tree */ int max_code; /* largest code with non zero frequency */ - static_tree_desc *stat_desc; /* the corresponding static tree */ + const static_tree_desc *stat_desc; /* the corresponding static tree */ } FAR tree_desc; typedef ush Pos; @@ -100,10 +103,10 @@ typedef struct internal_state { Bytef *pending_buf; /* output still pending */ ulg pending_buf_size; /* size of pending_buf */ Bytef *pending_out; /* next pending byte to output to the stream */ - uInt pending; /* nb of bytes in the pending buffer */ + ulg pending; /* nb of bytes in the pending buffer */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ - uInt gzindex; /* where in extra, name, or comment */ + ulg gzindex; /* where in extra, name, or comment */ Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ @@ -249,7 +252,7 @@ typedef struct internal_state { uInt matches; /* number of string matches in current block */ uInt insert; /* bytes at end of window left to insert */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif @@ -275,7 +278,7 @@ typedef struct internal_state { /* Output a byte on the stream. * IN assertion: there is enough room in pending_buf. */ -#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} +#define put_byte(s, c) {s->pending_buf[s->pending++] = (Bytef)(c);} #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) @@ -309,7 +312,7 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, * used. */ -#ifndef DEBUG +#ifndef ZLIB_DEBUG /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) @@ -328,8 +331,8 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, flush = (s->last_lit == s->lit_bufsize-1); \ } # define _tr_tally_dist(s, distance, length, flush) \ - { uch len = (length); \ - ush dist = (distance); \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ s->d_buf[s->last_lit] = dist; \ s->l_buf[s->last_lit++] = len; \ dist--; \ diff --git a/contrib/zlib/gzguts.h b/contrib/zlib/gzguts.h index d87659d03..6378d468a 100644 --- a/contrib/zlib/gzguts.h +++ b/contrib/zlib/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -25,6 +25,10 @@ # include # include #endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif #include #ifdef _WIN32 @@ -35,6 +39,10 @@ # include #endif +#if defined(_WIN32) +# define WIDECHAR +#endif + #ifdef WINAPI_FAMILY # define open _open # define read _read @@ -95,18 +103,19 @@ # endif #endif -/* unlike snprintf (which is required in C99, yet still not supported by - Microsoft more than a decade later!), _snprintf does not guarantee null - termination of the result -- however this is only used in gzlib.c where +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where the result is assured to fit in the space provided */ -#ifdef _MSC_VER +#if defined(_MSC_VER) && _MSC_VER < 1900 # define snprintf _snprintf #endif #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ /* gz* functions always use library allocation functions */ #ifndef STDC @@ -170,7 +179,7 @@ typedef struct { char *path; /* path or fd for error messages */ unsigned size; /* buffer size, zero if not allocated yet */ unsigned want; /* requested buffer size, default is GZBUFSIZE */ - unsigned char *in; /* input buffer */ + unsigned char *in; /* input buffer (double-sized when writing) */ unsigned char *out; /* output buffer (double-sized when reading) */ int direct; /* 0 if processing gzip, 1 if transparent */ /* just for reading */ diff --git a/contrib/zlib/gzlib.c b/contrib/zlib/gzlib.c index fae202ef8..4838bf047 100644 --- a/contrib/zlib/gzlib.c +++ b/contrib/zlib/gzlib.c @@ -1,5 +1,5 @@ /* gzlib.c -- zlib functions common to reading and writing gzip files - * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler + * Copyright (C) 2004-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -94,7 +94,7 @@ local gzFile gz_open(path, fd, mode) const char *mode; { gz_statep state; - size_t len; + z_size_t len; int oflag; #ifdef O_CLOEXEC int cloexec = 0; @@ -188,10 +188,10 @@ local gzFile gz_open(path, fd, mode) } /* save the path name for error messages */ -#ifdef _WIN32 +#ifdef WIDECHAR if (fd == -2) { len = wcstombs(NULL, path, 0); - if (len == (size_t)-1) + if (len == (z_size_t)-1) len = 0; } else @@ -202,7 +202,7 @@ local gzFile gz_open(path, fd, mode) free(state); return NULL; } -#ifdef _WIN32 +#ifdef WIDECHAR if (fd == -2) if (len) wcstombs(state->path, path, len + 1); @@ -211,7 +211,7 @@ local gzFile gz_open(path, fd, mode) else #endif #if !defined(NO_snprintf) && !defined(NO_vsnprintf) - snprintf(state->path, len + 1, "%s", (const char *)path); + (void)snprintf(state->path, len + 1, "%s", (const char *)path); #else strcpy(state->path, path); #endif @@ -239,7 +239,7 @@ local gzFile gz_open(path, fd, mode) /* open the file with the appropriate flags (or just use fd) */ state->fd = fd > -1 ? fd : ( -#ifdef _WIN32 +#ifdef WIDECHAR fd == -2 ? _wopen(path, oflag, 0666) : #endif open((const char *)path, oflag, 0666)); @@ -248,8 +248,10 @@ local gzFile gz_open(path, fd, mode) free(state); return NULL; } - if (state->mode == GZ_APPEND) + if (state->mode == GZ_APPEND) { + LSEEK(state->fd, 0, SEEK_END); /* so gzoffset() is correct */ state->mode = GZ_WRITE; /* simplify later checks */ + } /* save the current position for rewinding (only if reading) */ if (state->mode == GZ_READ) { @@ -291,7 +293,7 @@ gzFile ZEXPORT gzdopen(fd, mode) if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) return NULL; #if !defined(NO_snprintf) && !defined(NO_vsnprintf) - snprintf(path, 7 + 3 * sizeof(int), "", fd); /* for debugging */ + (void)snprintf(path, 7 + 3 * sizeof(int), "", fd); #else sprintf(path, "", fd); /* for debugging */ #endif @@ -301,7 +303,7 @@ gzFile ZEXPORT gzdopen(fd, mode) } /* -- see zlib.h -- */ -#ifdef _WIN32 +#ifdef WIDECHAR gzFile ZEXPORT gzopen_w(path, mode) const wchar_t *path; const char *mode; @@ -329,6 +331,8 @@ int ZEXPORT gzbuffer(file, size) return -1; /* check and set requested size */ + if ((size << 1) < size) + return -1; /* need to be able to double it */ if (size < 2) size = 2; /* need two bytes to check magic header */ state->want = size; @@ -393,7 +397,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) /* if within raw area while reading, just go there */ if (state->mode == GZ_READ && state->how == COPY && state->x.pos + offset >= 0) { - ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR); + ret = LSEEK(state->fd, offset - (z_off64_t)state->x.have, SEEK_CUR); if (ret == -1) return -1; state->x.have = 0; @@ -604,14 +608,13 @@ void ZLIB_INTERNAL gz_error(state, err, msg) return; } #if !defined(NO_snprintf) && !defined(NO_vsnprintf) - snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, - "%s%s%s", state->path, ": ", msg); + (void)snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, + "%s%s%s", state->path, ": ", msg); #else strcpy(state->msg, state->path); strcat(state->msg, ": "); strcat(state->msg, msg); #endif - return; } #ifndef INT_MAX diff --git a/contrib/zlib/gzread.c b/contrib/zlib/gzread.c index bf4538eb2..f94abfed3 100644 --- a/contrib/zlib/gzread.c +++ b/contrib/zlib/gzread.c @@ -1,5 +1,5 @@ /* gzread.c -- zlib functions for reading gzip files - * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -12,6 +12,7 @@ local int gz_look OF((gz_statep)); local int gz_decomp OF((gz_statep)); local int gz_fetch OF((gz_statep)); local int gz_skip OF((gz_statep, z_off64_t)); +local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); /* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from state->fd, and update state->eof, state->err, and state->msg as appropriate. @@ -24,13 +25,17 @@ local int gz_load(state, buf, len, have) unsigned *have; { int ret; + unsigned get, max = ((unsigned)-1 >> 2) + 1; *have = 0; do { - ret = read(state->fd, buf + *have, len - *have); + get = len - *have; + if (get > max) + get = max; + ret = read(state->fd, buf + *have, get); if (ret <= 0) break; - *have += ret; + *have += (unsigned)ret; } while (*have < len); if (ret < 0) { gz_error(state, Z_ERRNO, zstrerror()); @@ -94,10 +99,8 @@ local int gz_look(state) state->in = (unsigned char *)malloc(state->want); state->out = (unsigned char *)malloc(state->want << 1); if (state->in == NULL || state->out == NULL) { - if (state->out != NULL) - free(state->out); - if (state->in != NULL) - free(state->in); + free(state->out); + free(state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } @@ -284,33 +287,17 @@ local int gz_skip(state, len) return 0; } -/* -- see zlib.h -- */ -int ZEXPORT gzread(file, buf, len) - gzFile file; - voidp buf; - unsigned len; -{ - unsigned got, n; +/* Read len bytes into buf from file, or less than len up to the end of the + input. Return the number of bytes read. If zero is returned, either the + end of file was reached, or there was an error. state->err must be + consulted in that case to determine which. */ +local z_size_t gz_read(state, buf, len) gz_statep state; - z_streamp strm; - - /* get internal structure */ - if (file == NULL) - return -1; - state = (gz_statep)file; - strm = &(state->strm); - - /* check that we're reading and that there's no (serious) error */ - if (state->mode != GZ_READ || - (state->err != Z_OK && state->err != Z_BUF_ERROR)) - return -1; - - /* since an int is returned, make sure len fits in one, otherwise return - with an error (this avoids the flaw in the interface) */ - if ((int)len < 0) { - gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); - return -1; - } + voidp buf; + z_size_t len; +{ + z_size_t got; + unsigned n; /* if len is zero, avoid unnecessary operations */ if (len == 0) @@ -320,32 +307,38 @@ int ZEXPORT gzread(file, buf, len) if (state->seek) { state->seek = 0; if (gz_skip(state, state->skip) == -1) - return -1; + return 0; } /* get len bytes to buf, or less than len if at the end */ got = 0; do { + /* set n to the maximum amount of len that fits in an unsigned int */ + n = (unsigned)-1; + if (n > len) + n = (unsigned)len; + /* first just try copying data from the output buffer */ if (state->x.have) { - n = state->x.have > len ? len : state->x.have; + if (state->x.have < n) + n = state->x.have; memcpy(buf, state->x.next, n); state->x.next += n; state->x.have -= n; } /* output buffer empty -- return if we're at the end of the input */ - else if (state->eof && strm->avail_in == 0) { + else if (state->eof && state->strm.avail_in == 0) { state->past = 1; /* tried to read past end */ break; } /* need output data -- for small len or new stream load up our output buffer */ - else if (state->how == LOOK || len < (state->size << 1)) { + else if (state->how == LOOK || n < (state->size << 1)) { /* get more output, looking for header if required */ if (gz_fetch(state) == -1) - return -1; + return 0; continue; /* no progress yet -- go back to copy above */ /* the copy above assures that we will leave with space in the output buffer, allowing at least one gzungetc() to succeed */ @@ -353,16 +346,16 @@ int ZEXPORT gzread(file, buf, len) /* large len -- read directly into user buffer */ else if (state->how == COPY) { /* read directly */ - if (gz_load(state, (unsigned char *)buf, len, &n) == -1) - return -1; + if (gz_load(state, (unsigned char *)buf, n, &n) == -1) + return 0; } /* large len -- decompress directly into user buffer */ else { /* state->how == GZIP */ - strm->avail_out = len; - strm->next_out = (unsigned char *)buf; + state->strm.avail_out = n; + state->strm.next_out = (unsigned char *)buf; if (gz_decomp(state) == -1) - return -1; + return 0; n = state->x.have; state->x.have = 0; } @@ -374,8 +367,75 @@ int ZEXPORT gzread(file, buf, len) state->x.pos += n; } while (len); - /* return number of bytes read into user buffer (will fit in int) */ - return (int)got; + /* return number of bytes read into user buffer */ + return got; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in an int"); + return -1; + } + + /* read len or fewer bytes to buf */ + len = (unsigned)gz_read(state, buf, len); + + /* check for an error */ + if (len == 0 && state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* return the number of bytes read (this is assured to fit in an int) */ + return (int)len; +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfread(buf, size, nitems, file) + voidp buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* read len or fewer bytes to buf, return the number of full items read */ + return len ? gz_read(state, buf, len) / size : 0; } /* -- see zlib.h -- */ @@ -387,7 +447,6 @@ int ZEXPORT gzread(file, buf, len) int ZEXPORT gzgetc(file) gzFile file; { - int ret; unsigned char buf[1]; gz_statep state; @@ -408,9 +467,8 @@ int ZEXPORT gzgetc(file) return *(state->x.next)++; } - /* nothing there -- try gzread() */ - ret = gzread(file, buf, 1); - return ret < 1 ? -1 : buf[0]; + /* nothing there -- try gz_read() */ + return gz_read(state, buf, 1) < 1 ? -1 : buf[0]; } int ZEXPORT gzgetc_(file) @@ -451,7 +509,7 @@ int ZEXPORT gzungetc(c, file) if (state->x.have == 0) { state->x.have = 1; state->x.next = state->out + (state->size << 1) - 1; - state->x.next[0] = c; + state->x.next[0] = (unsigned char)c; state->x.pos--; state->past = 0; return c; @@ -473,7 +531,7 @@ int ZEXPORT gzungetc(c, file) } state->x.have++; state->x.next--; - state->x.next[0] = c; + state->x.next[0] = (unsigned char)c; state->x.pos--; state->past = 0; return c; diff --git a/contrib/zlib/gzwrite.c b/contrib/zlib/gzwrite.c index 3ed161f44..3560193b8 100644 --- a/contrib/zlib/gzwrite.c +++ b/contrib/zlib/gzwrite.c @@ -1,5 +1,5 @@ /* gzwrite.c -- zlib functions for writing gzip files - * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * Copyright (C) 2004-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,17 +9,19 @@ local int gz_init OF((gz_statep)); local int gz_comp OF((gz_statep, int)); local int gz_zero OF((gz_statep, z_off64_t)); +local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); /* Initialize state for writing a gzip file. Mark initialization by setting - state->size to non-zero. Return -1 on failure or 0 on success. */ + state->size to non-zero. Return -1 on a memory allocation failure, or 0 on + success. */ local int gz_init(state) gz_statep state; { int ret; z_streamp strm = &(state->strm); - /* allocate input buffer */ - state->in = (unsigned char *)malloc(state->want); + /* allocate input buffer (double size for gzprintf) */ + state->in = (unsigned char *)malloc(state->want << 1); if (state->in == NULL) { gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; @@ -47,6 +49,7 @@ local int gz_init(state) gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } + strm->next_in = NULL; } /* mark state as initialized */ @@ -62,17 +65,17 @@ local int gz_init(state) } /* Compress whatever is at avail_in and next_in and write to the output file. - Return -1 if there is an error writing to the output file, otherwise 0. - flush is assumed to be a valid deflate() flush value. If flush is Z_FINISH, - then the deflate() state is reset to start a new gzip stream. If gz->direct - is true, then simply write to the output file without compressing, and - ignore flush. */ + Return -1 if there is an error writing to the output file or if gz_init() + fails to allocate memory, otherwise 0. flush is assumed to be a valid + deflate() flush value. If flush is Z_FINISH, then the deflate() state is + reset to start a new gzip stream. If gz->direct is true, then simply write + to the output file without compressing, and ignore flush. */ local int gz_comp(state, flush) gz_statep state; int flush; { - int ret, got; - unsigned have; + int ret, writ; + unsigned have, put, max = ((unsigned)-1 >> 2) + 1; z_streamp strm = &(state->strm); /* allocate memory if this is the first time through */ @@ -81,12 +84,16 @@ local int gz_comp(state, flush) /* write directly if requested */ if (state->direct) { - got = write(state->fd, strm->next_in, strm->avail_in); - if (got < 0 || (unsigned)got != strm->avail_in) { - gz_error(state, Z_ERRNO, zstrerror()); - return -1; + while (strm->avail_in) { + put = strm->avail_in > max ? max : strm->avail_in; + writ = write(state->fd, strm->next_in, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + strm->avail_in -= (unsigned)writ; + strm->next_in += writ; } - strm->avail_in = 0; return 0; } @@ -97,17 +104,21 @@ local int gz_comp(state, flush) doing Z_FINISH then don't write until we get to Z_STREAM_END */ if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && (flush != Z_FINISH || ret == Z_STREAM_END))) { - have = (unsigned)(strm->next_out - state->x.next); - if (have && ((got = write(state->fd, state->x.next, have)) < 0 || - (unsigned)got != have)) { - gz_error(state, Z_ERRNO, zstrerror()); - return -1; + while (strm->next_out > state->x.next) { + put = strm->next_out - state->x.next > (int)max ? max : + (unsigned)(strm->next_out - state->x.next); + writ = write(state->fd, state->x.next, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + state->x.next += writ; } if (strm->avail_out == 0) { strm->avail_out = state->size; strm->next_out = state->out; + state->x.next = state->out; } - state->x.next = strm->next_out; } /* compress */ @@ -129,7 +140,8 @@ local int gz_comp(state, flush) return 0; } -/* Compress len zeros to output. Return -1 on error, 0 on success. */ +/* Compress len zeros to output. Return -1 on a write error or memory + allocation failure by gz_comp(), or 0 on success. */ local int gz_zero(state, len) gz_statep state; z_off64_t len; @@ -161,32 +173,14 @@ local int gz_zero(state, len) return 0; } -/* -- see zlib.h -- */ -int ZEXPORT gzwrite(file, buf, len) - gzFile file; - voidpc buf; - unsigned len; -{ - unsigned put = len; +/* Write len bytes from buf to file. Return the number of bytes written. If + the returned value is less than len, then there was an error. */ +local z_size_t gz_write(state, buf, len) gz_statep state; - z_streamp strm; - - /* get internal structure */ - if (file == NULL) - return 0; - state = (gz_statep)file; - strm = &(state->strm); - - /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) - return 0; - - /* since an int is returned, make sure len fits in one, otherwise return - with an error (this avoids the flaw in the interface) */ - if ((int)len < 0) { - gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); - return 0; - } + voidpc buf; + z_size_t len; +{ + z_size_t put = len; /* if len is zero, avoid unnecessary operations */ if (len == 0) @@ -209,14 +203,15 @@ int ZEXPORT gzwrite(file, buf, len) do { unsigned have, copy; - if (strm->avail_in == 0) - strm->next_in = state->in; - have = (unsigned)((strm->next_in + strm->avail_in) - state->in); + if (state->strm.avail_in == 0) + state->strm.next_in = state->in; + have = (unsigned)((state->strm.next_in + state->strm.avail_in) - + state->in); copy = state->size - have; if (copy > len) - copy = len; + copy = (unsigned)len; memcpy(state->in + have, buf, copy); - strm->avail_in += copy; + state->strm.avail_in += copy; state->x.pos += copy; buf = (const char *)buf + copy; len -= copy; @@ -226,19 +221,83 @@ int ZEXPORT gzwrite(file, buf, len) } else { /* consume whatever's left in the input buffer */ - if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + if (state->strm.avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return 0; /* directly compress user buffer to file */ - strm->avail_in = len; - strm->next_in = (z_const Bytef *)buf; - state->x.pos += len; - if (gz_comp(state, Z_NO_FLUSH) == -1) - return 0; + state->strm.next_in = (z_const Bytef *)buf; + do { + unsigned n = (unsigned)-1; + if (n > len) + n = (unsigned)len; + state->strm.avail_in = n; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + len -= n; + } while (len); } - /* input was all buffered or compressed (put will fit in int) */ - return (int)put; + /* input was all buffered or compressed */ + return put; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return 0; + } + + /* write len bytes from buf (the return value will fit in an int) */ + return (int)gz_write(state, buf, len); +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) + voidpc buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* write len bytes to buf, return the number of full items written */ + return len ? gz_write(state, buf, len) / size : 0; } /* -- see zlib.h -- */ @@ -275,7 +334,7 @@ int ZEXPORT gzputc(file, c) strm->next_in = state->in; have = (unsigned)((strm->next_in + strm->avail_in) - state->in); if (have < state->size) { - state->in[have] = c; + state->in[have] = (unsigned char)c; strm->avail_in++; state->x.pos++; return c & 0xff; @@ -283,8 +342,8 @@ int ZEXPORT gzputc(file, c) } /* no room in buffer or not initialized, use gz_write() */ - buf[0] = c; - if (gzwrite(file, buf, 1) != 1) + buf[0] = (unsigned char)c; + if (gz_write(state, buf, 1) != 1) return -1; return c & 0xff; } @@ -294,13 +353,26 @@ int ZEXPORT gzputs(file, str) gzFile file; const char *str; { - int ret; - unsigned len; + z_size_t len, put; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; /* write string */ - len = (unsigned)strlen(str); - ret = gzwrite(file, str, len); - return ret == 0 && len != 0 ? -1 : ret; + len = strlen(str); + if ((int)len < 0 || (unsigned)len != len) { + gz_error(state, Z_STREAM_ERROR, "string length does not fit in int"); + return -1; + } + put = gz_write(state, str, len); + return put < len ? -1 : (int)len; } #if defined(STDC) || defined(Z_HAVE_STDARG_H) @@ -309,63 +381,73 @@ int ZEXPORT gzputs(file, str) /* -- see zlib.h -- */ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { - int size, len; + int len; + unsigned left; + char *next; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) - return -1; + return Z_STREAM_ERROR; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) - return 0; + return Z_STREAM_ERROR; /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) - return 0; + return state->err; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return 0; + return state->err; } - /* consume whatever's left in the input buffer */ - if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) - return 0; - - /* do the printf() into the input buffer, put length in len */ - size = (int)(state->size); - state->in[size - 1] = 0; + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(state->in + (strm->next_in - state->in) + strm->avail_in); + next[state->size - 1] = 0; #ifdef NO_vsnprintf # ifdef HAS_vsprintf_void - (void)vsprintf((char *)(state->in), format, va); - for (len = 0; len < size; len++) - if (state->in[len] == 0) break; + (void)vsprintf(next, format, va); + for (len = 0; len < state->size; len++) + if (next[len] == 0) break; # else - len = vsprintf((char *)(state->in), format, va); + len = vsprintf(next, format, va); # endif #else # ifdef HAS_vsnprintf_void - (void)vsnprintf((char *)(state->in), size, format, va); - len = strlen((char *)(state->in)); + (void)vsnprintf(next, state->size, format, va); + len = strlen(next); # else - len = vsnprintf((char *)(state->in), size, format, va); + len = vsnprintf(next, state->size, format, va); # endif #endif /* check that printf() results fit in buffer */ - if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) + if (len == 0 || (unsigned)len >= state->size || next[state->size - 1] != 0) return 0; - /* update buffer and position, defer compression until needed */ - strm->avail_in = (unsigned)len; - strm->next_in = state->in; + /* update buffer and position, compress first half if past that */ + strm->avail_in += (unsigned)len; state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } return len; } @@ -390,73 +472,82 @@ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; { - int size, len; + unsigned len, left; + char *next; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) - return -1; + return Z_STREAM_ERROR; state = (gz_statep)file; strm = &(state->strm); /* check that can really pass pointer in ints */ if (sizeof(int) != sizeof(void *)) - return 0; + return Z_STREAM_ERROR; /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) - return 0; + return Z_STREAM_ERROR; /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) - return 0; + return state->error; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return 0; + return state->error; } - /* consume whatever's left in the input buffer */ - if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) - return 0; - - /* do the printf() into the input buffer, put length in len */ - size = (int)(state->size); - state->in[size - 1] = 0; + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(strm->next_in + strm->avail_in); + next[state->size - 1] = 0; #ifdef NO_snprintf # ifdef HAS_sprintf_void - sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + a13, a14, a15, a16, a17, a18, a19, a20); for (len = 0; len < size; len++) - if (state->in[len] == 0) break; + if (next[len] == 0) + break; # else - len = sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, + a12, a13, a14, a15, a16, a17, a18, a19, a20); # endif #else # ifdef HAS_snprintf_void - snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - len = strlen((char *)(state->in)); + snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, + a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = strlen(next); # else - len = snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, - a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, - a19, a20); + len = snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); # endif #endif /* check that printf() results fit in buffer */ - if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) + if (len == 0 || len >= state->size || next[state->size - 1] != 0) return 0; - /* update buffer and position, defer compression until needed */ - strm->avail_in = (unsigned)len; - strm->next_in = state->in; + /* update buffer and position, compress first half if past that */ + strm->avail_in += len; state->x.pos += len; - return len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } + return (int)len; } #endif @@ -470,7 +561,7 @@ int ZEXPORT gzflush(file, flush) /* get internal structure */ if (file == NULL) - return -1; + return Z_STREAM_ERROR; state = (gz_statep)file; /* check that we're writing and that there's no error */ @@ -485,11 +576,11 @@ int ZEXPORT gzflush(file, flush) if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return -1; + return state->err; } /* compress remaining data with requested flush */ - gz_comp(state, flush); + (void)gz_comp(state, flush); return state->err; } @@ -520,13 +611,13 @@ int ZEXPORT gzsetparams(file, level, strategy) if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) - return -1; + return state->err; } /* change compression parameters for subsequent input */ if (state->size) { /* flush previous input with previous parameters before changing */ - if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1) + if (strm->avail_in && gz_comp(state, Z_BLOCK) == -1) return state->err; deflateParams(strm, level, strategy); } diff --git a/contrib/zlib/infback.c b/contrib/zlib/infback.c index f3833c2e4..59679ecbf 100644 --- a/contrib/zlib/infback.c +++ b/contrib/zlib/infback.c @@ -1,5 +1,5 @@ /* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2011 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -61,7 +61,7 @@ int stream_size; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->dmax = 32768U; - state->wbits = windowBits; + state->wbits = (uInt)windowBits; state->wsize = 1U << windowBits; state->window = window; state->wnext = 0; diff --git a/contrib/zlib/inffast.c b/contrib/zlib/inffast.c index bda59ceb6..1fec7f363 100644 --- a/contrib/zlib/inffast.c +++ b/contrib/zlib/inffast.c @@ -1,5 +1,5 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2008, 2010, 2013 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,26 +8,9 @@ #include "inflate.h" #include "inffast.h" -#ifndef ASMINF - -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") #else -# define OFF 1 -# define PUP(a) *++(a) -#endif /* Decode literal, length, and distance codes and write out the resulting @@ -87,7 +70,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ code const FAR *dcode; /* local strm->distcode */ unsigned lmask; /* mask for first level of length codes */ unsigned dmask; /* mask for first level of distance codes */ - code here; /* retrieved table entry */ + code const *here; /* retrieved table entry */ unsigned op; /* code bits, operation, extra bits, or */ /* window position, window bytes to copy */ unsigned len; /* match length, unused bytes */ @@ -96,9 +79,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; - in = strm->next_in - OFF; + in = strm->next_in; last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + out = strm->next_out; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT @@ -119,29 +102,29 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } - here = lcode[hold & lmask]; + here = lcode + (hold & lmask); dolen: - op = (unsigned)(here.bits); + op = (unsigned)(here->bits); hold >>= op; bits -= op; - op = (unsigned)(here.op); + op = (unsigned)(here->op); if (op == 0) { /* literal */ - Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ? "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", here.val)); - PUP(out) = (unsigned char)(here.val); + "inflate: literal 0x%02x\n", here->val)); + *out++ = (unsigned char)(here->val); } else if (op & 16) { /* length base */ - len = (unsigned)(here.val); + len = (unsigned)(here->val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -150,25 +133,25 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } - here = dcode[hold & dmask]; + here = dcode + (hold & dmask); dodist: - op = (unsigned)(here.bits); + op = (unsigned)(here->bits); hold >>= op; bits -= op; - op = (unsigned)(here.op); + op = (unsigned)(here->op); if (op & 16) { /* distance base */ - dist = (unsigned)(here.val); + dist = (unsigned)(here->val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -196,30 +179,30 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { - PUP(out) = 0; + *out++ = 0; } while (--len); continue; } len -= op - whave; do { - PUP(out) = 0; + *out++ = 0; } while (--op > whave); if (op == 0) { from = out - dist; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--len); continue; } #endif } - from = window - OFF; + from = window; if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -230,14 +213,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; + from = window; if (wnext < len) { /* some from start of window */ op = wnext; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -248,40 +231,40 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } while (len > 2); if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } } else if ((op & 64) == 0) { /* 2nd level distance code */ - here = dcode[here.val + (hold & ((1U << op) - 1))]; + here = dcode + here->val + (hold & ((1U << op) - 1)); goto dodist; } else { @@ -291,7 +274,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } } else if ((op & 64) == 0) { /* 2nd level length code */ - here = lcode[here.val + (hold & ((1U << op) - 1))]; + here = lcode + here->val + (hold & ((1U << op) - 1)); goto dolen; } else if (op & 32) { /* end-of-block */ @@ -313,8 +296,8 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; + strm->next_in = in; + strm->next_out = out; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); diff --git a/contrib/zlib/inflate.c b/contrib/zlib/inflate.c index 870f89bb4..575fcdf82 100644 --- a/contrib/zlib/inflate.c +++ b/contrib/zlib/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2012 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -92,6 +92,7 @@ #endif /* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); local void fixedtables OF((struct inflate_state FAR *state)); local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, unsigned copy)); @@ -101,12 +102,26 @@ local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + int ZEXPORT inflateResetKeep(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; strm->total_in = strm->total_out = state->total = 0; strm->msg = Z_NULL; @@ -131,7 +146,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; state->wsize = 0; state->whave = 0; @@ -147,7 +162,7 @@ int windowBits; struct inflate_state FAR *state; /* get the state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* extract wrap request from windowBits parameter */ @@ -156,7 +171,7 @@ int windowBits; windowBits = -windowBits; } else { - wrap = (windowBits >> 4) + 1; + wrap = (windowBits >> 4) + 5; #ifdef GUNZIP if (windowBits < 48) windowBits &= 15; @@ -210,7 +225,9 @@ int stream_size; if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; + state->strm = strm; state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { ZFREE(strm, state); @@ -234,17 +251,17 @@ int value; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; state->bits = 0; return Z_OK; } - if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += value << state->bits; - state->bits += bits; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; return Z_OK; } @@ -625,7 +642,7 @@ int flush; static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - if (strm == Z_NULL || strm->state == Z_NULL || strm->next_out == Z_NULL || + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0)) return Z_STREAM_ERROR; @@ -645,6 +662,8 @@ int flush; NEEDBITS(16); #ifdef GUNZIP if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; state->check = crc32(0L, Z_NULL, 0); CRC2(state->check, hold); INITBITS(); @@ -672,7 +691,7 @@ int flush; len = BITS(4) + 8; if (state->wbits == 0) state->wbits = len; - else if (len > state->wbits) { + if (len > 15 || len > state->wbits) { strm->msg = (char *)"invalid window size"; state->mode = BAD; break; @@ -699,14 +718,16 @@ int flush; } if (state->head != Z_NULL) state->head->text = (int)((hold >> 8) & 1); - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = TIME; case TIME: NEEDBITS(32); if (state->head != Z_NULL) state->head->time = hold; - if (state->flags & 0x0200) CRC4(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); INITBITS(); state->mode = OS; case OS: @@ -715,7 +736,8 @@ int flush; state->head->xflags = (int)(hold & 0xff); state->head->os = (int)(hold >> 8); } - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = EXLEN; case EXLEN: @@ -724,7 +746,8 @@ int flush; state->length = (unsigned)(hold); if (state->head != Z_NULL) state->head->extra_len = (unsigned)hold; - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); } else if (state->head != Z_NULL) @@ -742,7 +765,7 @@ int flush; len + copy > state->head->extra_max ? state->head->extra_max - len : copy); } - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -761,9 +784,9 @@ int flush; if (state->head != Z_NULL && state->head->name != Z_NULL && state->length < state->head->name_max) - state->head->name[state->length++] = len; + state->head->name[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -782,9 +805,9 @@ int flush; if (state->head != Z_NULL && state->head->comment != Z_NULL && state->length < state->head->comm_max) - state->head->comment[state->length++] = len; + state->head->comment[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -796,7 +819,7 @@ int flush; case HCRC: if (state->flags & 0x0200) { NEEDBITS(16); - if (hold != (state->check & 0xffff)) { + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { strm->msg = (char *)"header crc mismatch"; state->mode = BAD; break; @@ -1177,11 +1200,11 @@ int flush; out -= left; strm->total_out += out; state->total += out; - if (out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; - if (( + if ((state->wrap & 4) && ( #ifdef GUNZIP state->flags ? hold : #endif @@ -1240,10 +1263,10 @@ int flush; strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); - strm->data_type = state->bits + (state->last ? 64 : 0) + + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + (state->mode == TYPE ? 128 : 0) + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) @@ -1255,7 +1278,7 @@ int ZEXPORT inflateEnd(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->window != Z_NULL) ZFREE(strm, state->window); @@ -1273,7 +1296,7 @@ uInt *dictLength; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* copy dictionary */ @@ -1298,7 +1321,7 @@ uInt dictLength; int ret; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->wrap != 0 && state->mode != DICT) return Z_STREAM_ERROR; @@ -1330,7 +1353,7 @@ gz_headerp head; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; @@ -1383,7 +1406,7 @@ z_streamp strm; struct inflate_state FAR *state; /* check parameters */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; @@ -1410,6 +1433,8 @@ z_streamp strm; /* return no joy or set up to restart inflate() on a new block */ if (state->have != 4) return Z_DATA_ERROR; + if (state->mode == HEAD) + state->wrap = 0; /* never processed header, so assume raw */ in = strm->total_in; out = strm->total_out; inflateReset(strm); strm->total_in = in; strm->total_out = out; @@ -1430,7 +1455,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } @@ -1445,8 +1470,7 @@ z_streamp source; unsigned wsize; /* check input */ - if (dest == Z_NULL || source == Z_NULL || source->state == Z_NULL || - source->zalloc == (alloc_func)0 || source->zfree == (free_func)0) + if (inflateStateCheck(source) || dest == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)source->state; @@ -1467,6 +1491,7 @@ z_streamp source; /* copy state */ zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { copy->lencode = copy->codes + (state->lencode - state->codes); @@ -1488,25 +1513,51 @@ int subvert; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - state->sane = !subvert; #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; return Z_OK; #else + (void)subvert; state->sane = 1; return Z_DATA_ERROR; #endif } +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check && state->wrap) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + long ZEXPORT inflateMark(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return -1L << 16; + if (inflateStateCheck(strm)) + return -(1L << 16); state = (struct inflate_state FAR *)strm->state; - return ((long)(state->back) << 16) + + return (long)(((unsigned long)((long)state->back)) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/contrib/zlib/inflate.h b/contrib/zlib/inflate.h index 95f4986d4..a46cce6b6 100644 --- a/contrib/zlib/inflate.h +++ b/contrib/zlib/inflate.h @@ -1,5 +1,5 @@ /* inflate.h -- internal inflate state definition - * Copyright (C) 1995-2009 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,7 +18,7 @@ /* Possible inflate modes between inflate() calls */ typedef enum { - HEAD, /* i: waiting for magic header */ + HEAD = 16180, /* i: waiting for magic header */ FLAGS, /* i: waiting for method and flags (gzip) */ TIME, /* i: waiting for modification time (gzip) */ OS, /* i: waiting for extra flags and operating system (gzip) */ @@ -77,11 +77,14 @@ typedef enum { CHECK -> LENGTH -> DONE */ -/* state maintained between inflate() calls. Approximately 10K bytes. */ +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ - int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ diff --git a/contrib/zlib/inftrees.c b/contrib/zlib/inftrees.c index 44d89cf24..716c98842 100644 --- a/contrib/zlib/inftrees.c +++ b/contrib/zlib/inftrees.c @@ -1,5 +1,5 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2013 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,7 +9,7 @@ #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.2.8 Copyright 1995-2013 Mark Adler "; + " inflate 1.2.11.1 Copyright 1995-2017 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -54,7 +54,7 @@ unsigned short FAR *work; code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ - int end; /* use base and extra for symbol > end */ + unsigned match; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -62,7 +62,7 @@ unsigned short FAR *work; 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 72, 78}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 198, 196}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, @@ -181,19 +181,17 @@ unsigned short FAR *work; switch (type) { case CODES: base = extra = work; /* dummy value--not used */ - end = 19; + match = 20; break; case LENS: base = lbase; - base -= 257; extra = lext; - extra -= 257; - end = 256; + match = 257; break; - default: /* DISTS */ + default: /* DISTS */ base = dbase; extra = dext; - end = -1; + match = 0; } /* initialize state for loop */ @@ -216,13 +214,13 @@ unsigned short FAR *work; for (;;) { /* create table entry */ here.bits = (unsigned char)(len - drop); - if ((int)(work[sym]) < end) { + if (work[sym] + 1U < match) { here.op = (unsigned char)0; here.val = work[sym]; } - else if ((int)(work[sym]) > end) { - here.op = (unsigned char)(extra[work[sym]]); - here.val = base[work[sym]]; + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; } else { here.op = (unsigned char)(32 + 64); /* end of block */ diff --git a/contrib/zlib/make_vms.com b/contrib/zlib/make_vms.com new file mode 100644 index 000000000..65e9d0cbc --- /dev/null +++ b/contrib/zlib/make_vms.com @@ -0,0 +1,867 @@ +$! make libz under VMS written by +$! Martin P.J. Zinser +$! +$! In case of problems with the install you might contact me at +$! zinser@zinser.no-ip.info(preferred) or +$! martin.zinser@eurexchange.com (work) +$! +$! Make procedure history for Zlib +$! +$!------------------------------------------------------------------------------ +$! Version history +$! 0.01 20060120 First version to receive a number +$! 0.02 20061008 Adapt to new Makefile.in +$! 0.03 20091224 Add support for large file check +$! 0.04 20100110 Add new gzclose, gzlib, gzread, gzwrite +$! 0.05 20100221 Exchange zlibdefs.h by zconf.h.in +$! 0.06 20120111 Fix missing amiss_err, update zconf_h.in, fix new exmples +$! subdir path, update module search in makefile.in +$! 0.07 20120115 Triggered by work done by Alexey Chupahin completly redesigned +$! shared image creation +$! 0.08 20120219 Make it work on VAX again, pre-load missing symbols to shared +$! image +$! 0.09 20120305 SMS. P1 sets builder ("MMK", "MMS", " " (built-in)). +$! "" -> automatic, preference: MMK, MMS, built-in. +$! +$ on error then goto err_exit +$! +$ true = 1 +$ false = 0 +$ tmpnam = "temp_" + f$getjpi("","pid") +$ tt = tmpnam + ".txt" +$ tc = tmpnam + ".c" +$ th = tmpnam + ".h" +$ define/nolog tconfig 'th' +$ its_decc = false +$ its_vaxc = false +$ its_gnuc = false +$ s_case = False +$! +$! Setup variables holding "config" information +$! +$ Make = "''p1'" +$ name = "Zlib" +$ version = "?.?.?" +$ v_string = "ZLIB_VERSION" +$ v_file = "zlib.h" +$ ccopt = "/include = []" +$ lopts = "" +$ dnsrl = "" +$ aconf_in_file = "zconf.h.in#zconf.h_in#zconf_h.in" +$ conf_check_string = "" +$ linkonly = false +$ optfile = name + ".opt" +$ mapfile = name + ".map" +$ libdefs = "" +$ vax = f$getsyi("HW_MODEL").lt.1024 +$ axp = f$getsyi("HW_MODEL").ge.1024 .and. f$getsyi("HW_MODEL").lt.4096 +$ ia64 = f$getsyi("HW_MODEL").ge.4096 +$! +$! 2012-03-05 SMS. +$! Why is this needed? And if it is needed, why not simply ".not. vax"? +$! +$!!! if axp .or. ia64 then set proc/parse=extended +$! +$ whoami = f$parse(f$environment("Procedure"),,,,"NO_CONCEAL") +$ mydef = F$parse(whoami,,,"DEVICE") +$ mydir = f$parse(whoami,,,"DIRECTORY") - "][" +$ myproc = f$parse(whoami,,,"Name") + f$parse(whoami,,,"type") +$! +$! Check for MMK/MMS +$! +$ if (Make .eqs. "") +$ then +$ If F$Search ("Sys$System:MMS.EXE") .nes. "" Then Make = "MMS" +$ If F$Type (MMK) .eqs. "STRING" Then Make = "MMK" +$ else +$ Make = f$edit( Make, "trim") +$ endif +$! +$ gosub find_version +$! +$ open/write topt tmp.opt +$ open/write optf 'optfile' +$! +$ gosub check_opts +$! +$! Look for the compiler used +$! +$ gosub check_compiler +$ close topt +$ close optf +$! +$ if its_decc +$ then +$ ccopt = "/prefix=all" + ccopt +$ if f$trnlnm("SYS") .eqs. "" +$ then +$ if axp +$ then +$ define sys sys$library: +$ else +$ ccopt = "/decc" + ccopt +$ define sys decc$library_include: +$ endif +$ endif +$! +$! 2012-03-05 SMS. +$! Why /NAMES = AS_IS? Why not simply ".not. vax"? And why not on VAX? +$! +$ if axp .or. ia64 +$ then +$ ccopt = ccopt + "/name=as_is/opt=(inline=speed)" +$ s_case = true +$ endif +$ endif +$ if its_vaxc .or. its_gnuc +$ then +$ if f$trnlnm("SYS").eqs."" then define sys sys$library: +$ endif +$! +$! Build a fake configure input header +$! +$ open/write conf_hin config.hin +$ write conf_hin "#undef _LARGEFILE64_SOURCE" +$ close conf_hin +$! +$! +$ i = 0 +$FIND_ACONF: +$ fname = f$element(i,"#",aconf_in_file) +$ if fname .eqs. "#" then goto AMISS_ERR +$ if f$search(fname) .eqs. "" +$ then +$ i = i + 1 +$ goto find_aconf +$ endif +$ open/read/err=aconf_err aconf_in 'fname' +$ open/write aconf zconf.h +$ACONF_LOOP: +$ read/end_of_file=aconf_exit aconf_in line +$ work = f$edit(line, "compress,trim") +$ if f$extract(0,6,work) .nes. "#undef" +$ then +$ if f$extract(0,12,work) .nes. "#cmakedefine" +$ then +$ write aconf line +$ endif +$ else +$ cdef = f$element(1," ",work) +$ gosub check_config +$ endif +$ goto aconf_loop +$ACONF_EXIT: +$ write aconf "" +$ write aconf "/* VMS specifics added by make_vms.com: */" +$ write aconf "#define VMS 1" +$ write aconf "#include " +$ write aconf "#include " +$ write aconf "#ifdef _LARGEFILE" +$ write aconf "# define off64_t __off64_t" +$ write aconf "# define fopen64 fopen" +$ write aconf "# define fseeko64 fseeko" +$ write aconf "# define lseek64 lseek" +$ write aconf "# define ftello64 ftell" +$ write aconf "#endif" +$ write aconf "#if !defined( __VAX) && (__CRTL_VER >= 70312000)" +$ write aconf "# define HAVE_VSNPRINTF" +$ write aconf "#endif" +$ close aconf_in +$ close aconf +$ if f$search("''th'") .nes. "" then delete 'th';* +$! Build the thing plain or with mms +$! +$ write sys$output "Compiling Zlib sources ..." +$ if make.eqs."" +$ then +$ if (f$search( "example.obj;*") .nes. "") then delete example.obj;* +$ if (f$search( "minigzip.obj;*") .nes. "") then delete minigzip.obj;* +$ CALL MAKE adler32.OBJ "CC ''CCOPT' adler32" - + adler32.c zlib.h zconf.h +$ CALL MAKE compress.OBJ "CC ''CCOPT' compress" - + compress.c zlib.h zconf.h +$ CALL MAKE crc32.OBJ "CC ''CCOPT' crc32" - + crc32.c zlib.h zconf.h +$ CALL MAKE deflate.OBJ "CC ''CCOPT' deflate" - + deflate.c deflate.h zutil.h zlib.h zconf.h +$ CALL MAKE gzclose.OBJ "CC ''CCOPT' gzclose" - + gzclose.c zutil.h zlib.h zconf.h +$ CALL MAKE gzlib.OBJ "CC ''CCOPT' gzlib" - + gzlib.c zutil.h zlib.h zconf.h +$ CALL MAKE gzread.OBJ "CC ''CCOPT' gzread" - + gzread.c zutil.h zlib.h zconf.h +$ CALL MAKE gzwrite.OBJ "CC ''CCOPT' gzwrite" - + gzwrite.c zutil.h zlib.h zconf.h +$ CALL MAKE infback.OBJ "CC ''CCOPT' infback" - + infback.c zutil.h inftrees.h inflate.h inffast.h inffixed.h +$ CALL MAKE inffast.OBJ "CC ''CCOPT' inffast" - + inffast.c zutil.h zlib.h zconf.h inffast.h +$ CALL MAKE inflate.OBJ "CC ''CCOPT' inflate" - + inflate.c zutil.h zlib.h zconf.h infblock.h +$ CALL MAKE inftrees.OBJ "CC ''CCOPT' inftrees" - + inftrees.c zutil.h zlib.h zconf.h inftrees.h +$ CALL MAKE trees.OBJ "CC ''CCOPT' trees" - + trees.c deflate.h zutil.h zlib.h zconf.h +$ CALL MAKE uncompr.OBJ "CC ''CCOPT' uncompr" - + uncompr.c zlib.h zconf.h +$ CALL MAKE zutil.OBJ "CC ''CCOPT' zutil" - + zutil.c zutil.h zlib.h zconf.h +$ write sys$output "Building Zlib ..." +$ CALL MAKE libz.OLB "lib/crea libz.olb *.obj" *.OBJ +$ write sys$output "Building example..." +$ CALL MAKE example.OBJ "CC ''CCOPT' [.test]example" - + [.test]example.c zlib.h zconf.h +$ call make example.exe "LINK example,libz.olb/lib" example.obj libz.olb +$ write sys$output "Building minigzip..." +$ CALL MAKE minigzip.OBJ "CC ''CCOPT' [.test]minigzip" - + [.test]minigzip.c zlib.h zconf.h +$ call make minigzip.exe - + "LINK minigzip,libz.olb/lib" - + minigzip.obj libz.olb +$ else +$ gosub crea_mms +$ write sys$output "Make ''name' ''version' with ''Make' " +$ 'make' +$ endif +$! +$! Create shareable image +$! +$ gosub crea_olist +$ write sys$output "Creating libzshr.exe" +$ call map_2_shopt 'mapfile' 'optfile' +$ LINK_'lopts'/SHARE=libzshr.exe modules.opt/opt,'optfile'/opt +$ write sys$output "Zlib build completed" +$ delete/nolog tmp.opt;* +$ exit +$AMISS_ERR: +$ write sys$output "No source for config.hin found." +$ write sys$output "Tried any of ''aconf_in_file'" +$ goto err_exit +$CC_ERR: +$ write sys$output "C compiler required to build ''name'" +$ goto err_exit +$ERR_EXIT: +$ set message/facil/ident/sever/text +$ close/nolog optf +$ close/nolog topt +$ close/nolog aconf_in +$ close/nolog aconf +$ close/nolog out +$ close/nolog min +$ close/nolog mod +$ close/nolog h_in +$ write sys$output "Exiting..." +$ exit 2 +$! +$! +$MAKE: SUBROUTINE !SUBROUTINE TO CHECK DEPENDENCIES +$ V = 'F$Verify(0) +$! P1 = What we are trying to make +$! P2 = Command to make it +$! P3 - P8 What it depends on +$ +$ If F$Search(P1) .Eqs. "" Then Goto Makeit +$ Time = F$CvTime(F$File(P1,"RDT")) +$arg=3 +$Loop: +$ Argument = P'arg +$ If Argument .Eqs. "" Then Goto Exit +$ El=0 +$Loop2: +$ File = F$Element(El," ",Argument) +$ If File .Eqs. " " Then Goto Endl +$ AFile = "" +$Loop3: +$ OFile = AFile +$ AFile = F$Search(File) +$ If AFile .Eqs. "" .Or. AFile .Eqs. OFile Then Goto NextEl +$ If F$CvTime(F$File(AFile,"RDT")) .Ges. Time Then Goto Makeit +$ Goto Loop3 +$NextEL: +$ El = El + 1 +$ Goto Loop2 +$EndL: +$ arg=arg+1 +$ If arg .Le. 8 Then Goto Loop +$ Goto Exit +$ +$Makeit: +$ VV=F$VERIFY(0) +$ write sys$output P2 +$ 'P2 +$ VV='F$Verify(VV) +$Exit: +$ If V Then Set Verify +$ENDSUBROUTINE +$!------------------------------------------------------------------------------ +$! +$! Check command line options and set symbols accordingly +$! +$!------------------------------------------------------------------------------ +$! Version history +$! 0.01 20041206 First version to receive a number +$! 0.02 20060126 Add new "HELP" target +$ CHECK_OPTS: +$ i = 1 +$ OPT_LOOP: +$ if i .lt. 9 +$ then +$ cparm = f$edit(p'i',"upcase") +$! +$! Check if parameter actually contains something +$! +$ if f$edit(cparm,"trim") .nes. "" +$ then +$ if cparm .eqs. "DEBUG" +$ then +$ ccopt = ccopt + "/noopt/deb" +$ lopts = lopts + "/deb" +$ endif +$ if f$locate("CCOPT=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ ccopt = ccopt + f$extract(start,len,cparm) +$ if f$locate("AS_IS",f$edit(ccopt,"UPCASE")) .lt. f$length(ccopt) - + then s_case = true +$ endif +$ if cparm .eqs. "LINK" then linkonly = true +$ if f$locate("LOPTS=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ lopts = lopts + f$extract(start,len,cparm) +$ endif +$ if f$locate("CC=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ cc_com = f$extract(start,len,cparm) + if (cc_com .nes. "DECC") .and. - + (cc_com .nes. "VAXC") .and. - + (cc_com .nes. "GNUC") +$ then +$ write sys$output "Unsupported compiler choice ''cc_com' ignored" +$ write sys$output "Use DECC, VAXC, or GNUC instead" +$ else +$ if cc_com .eqs. "DECC" then its_decc = true +$ if cc_com .eqs. "VAXC" then its_vaxc = true +$ if cc_com .eqs. "GNUC" then its_gnuc = true +$ endif +$ endif +$ if f$locate("MAKE=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ mmks = f$extract(start,len,cparm) +$ if (mmks .eqs. "MMK") .or. (mmks .eqs. "MMS") +$ then +$ make = mmks +$ else +$ write sys$output "Unsupported make choice ''mmks' ignored" +$ write sys$output "Use MMK or MMS instead" +$ endif +$ endif +$ if cparm .eqs. "HELP" then gosub bhelp +$ endif +$ i = i + 1 +$ goto opt_loop +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Look for the compiler used +$! +$! Version history +$! 0.01 20040223 First version to receive a number +$! 0.02 20040229 Save/set value of decc$no_rooted_search_lists +$! 0.03 20060202 Extend handling of GNU C +$! 0.04 20090402 Compaq -> hp +$CHECK_COMPILER: +$ if (.not. (its_decc .or. its_vaxc .or. its_gnuc)) +$ then +$ its_decc = (f$search("SYS$SYSTEM:DECC$COMPILER.EXE") .nes. "") +$ its_vaxc = .not. its_decc .and. (F$Search("SYS$System:VAXC.Exe") .nes. "") +$ its_gnuc = .not. (its_decc .or. its_vaxc) .and. (f$trnlnm("gnu_cc") .nes. "") +$ endif +$! +$! Exit if no compiler available +$! +$ if (.not. (its_decc .or. its_vaxc .or. its_gnuc)) +$ then goto CC_ERR +$ else +$ if its_decc +$ then +$ write sys$output "CC compiler check ... hp C" +$ if f$trnlnm("decc$no_rooted_search_lists") .nes. "" +$ then +$ dnrsl = f$trnlnm("decc$no_rooted_search_lists") +$ endif +$ define/nolog decc$no_rooted_search_lists 1 +$ else +$ if its_vaxc then write sys$output "CC compiler check ... VAX C" +$ if its_gnuc +$ then +$ write sys$output "CC compiler check ... GNU C" +$ if f$trnlnm(topt) then write topt "gnu_cc:[000000]gcclib.olb/lib" +$ if f$trnlnm(optf) then write optf "gnu_cc:[000000]gcclib.olb/lib" +$ cc = "gcc" +$ endif +$ if f$trnlnm(topt) then write topt "sys$share:vaxcrtl.exe/share" +$ if f$trnlnm(optf) then write optf "sys$share:vaxcrtl.exe/share" +$ endif +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! If MMS/MMK are available dump out the descrip.mms if required +$! +$CREA_MMS: +$ write sys$output "Creating descrip.mms..." +$ create descrip.mms +$ open/append out descrip.mms +$ copy sys$input: out +$ deck +# descrip.mms: MMS description file for building zlib on VMS +# written by Martin P.J. Zinser +# + +OBJS = adler32.obj, compress.obj, crc32.obj, gzclose.obj, gzlib.obj\ + gzread.obj, gzwrite.obj, uncompr.obj, infback.obj\ + deflate.obj, trees.obj, zutil.obj, inflate.obj, \ + inftrees.obj, inffast.obj + +$ eod +$ write out "CFLAGS=", ccopt +$ write out "LOPTS=", lopts +$ write out "all : example.exe minigzip.exe libz.olb" +$ copy sys$input: out +$ deck + @ write sys$output " Example applications available" + +libz.olb : libz.olb($(OBJS)) + @ write sys$output " libz available" + +example.exe : example.obj libz.olb + link $(LOPTS) example,libz.olb/lib + +minigzip.exe : minigzip.obj libz.olb + link $(LOPTS) minigzip,libz.olb/lib + +clean : + delete *.obj;*,libz.olb;*,*.opt;*,*.exe;* + + +# Other dependencies. +adler32.obj : adler32.c zutil.h zlib.h zconf.h +compress.obj : compress.c zlib.h zconf.h +crc32.obj : crc32.c zutil.h zlib.h zconf.h +deflate.obj : deflate.c deflate.h zutil.h zlib.h zconf.h +example.obj : [.test]example.c zlib.h zconf.h +gzclose.obj : gzclose.c zutil.h zlib.h zconf.h +gzlib.obj : gzlib.c zutil.h zlib.h zconf.h +gzread.obj : gzread.c zutil.h zlib.h zconf.h +gzwrite.obj : gzwrite.c zutil.h zlib.h zconf.h +inffast.obj : inffast.c zutil.h zlib.h zconf.h inftrees.h inffast.h +inflate.obj : inflate.c zutil.h zlib.h zconf.h +inftrees.obj : inftrees.c zutil.h zlib.h zconf.h inftrees.h +minigzip.obj : [.test]minigzip.c zlib.h zconf.h +trees.obj : trees.c deflate.h zutil.h zlib.h zconf.h +uncompr.obj : uncompr.c zlib.h zconf.h +zutil.obj : zutil.c zutil.h zlib.h zconf.h +infback.obj : infback.c zutil.h inftrees.h inflate.h inffast.h inffixed.h +$ eod +$ close out +$ return +$!------------------------------------------------------------------------------ +$! +$! Read list of core library sources from makefile.in and create options +$! needed to build shareable image +$! +$CREA_OLIST: +$ open/read min makefile.in +$ open/write mod modules.opt +$ src_check_list = "OBJZ =#OBJG =" +$MRLOOP: +$ read/end=mrdone min rec +$ i = 0 +$SRC_CHECK_LOOP: +$ src_check = f$element(i, "#", src_check_list) +$ i = i+1 +$ if src_check .eqs. "#" then goto mrloop +$ if (f$extract(0,6,rec) .nes. src_check) then goto src_check_loop +$ rec = rec - src_check +$ gosub extra_filnam +$ if (f$element(1,"\",rec) .eqs. "\") then goto mrloop +$MRSLOOP: +$ read/end=mrdone min rec +$ gosub extra_filnam +$ if (f$element(1,"\",rec) .nes. "\") then goto mrsloop +$MRDONE: +$ close min +$ close mod +$ return +$!------------------------------------------------------------------------------ +$! +$! Take record extracted in crea_olist and split it into single filenames +$! +$EXTRA_FILNAM: +$ myrec = f$edit(rec - "\", "trim,compress") +$ i = 0 +$FELOOP: +$ srcfil = f$element(i," ", myrec) +$ if (srcfil .nes. " ") +$ then +$ write mod f$parse(srcfil,,,"NAME"), ".obj" +$ i = i + 1 +$ goto feloop +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Find current Zlib version number +$! +$FIND_VERSION: +$ open/read h_in 'v_file' +$hloop: +$ read/end=hdone h_in rec +$ rec = f$edit(rec,"TRIM") +$ if (f$extract(0,1,rec) .nes. "#") then goto hloop +$ rec = f$edit(rec - "#", "TRIM") +$ if f$element(0," ",rec) .nes. "define" then goto hloop +$ if f$element(1," ",rec) .eqs. v_string +$ then +$ version = 'f$element(2," ",rec)' +$ goto hdone +$ endif +$ goto hloop +$hdone: +$ close h_in +$ return +$!------------------------------------------------------------------------------ +$! +$CHECK_CONFIG: +$! +$ in_ldef = f$locate(cdef,libdefs) +$ if (in_ldef .lt. f$length(libdefs)) +$ then +$ write aconf "#define ''cdef' 1" +$ libdefs = f$extract(0,in_ldef,libdefs) + - + f$extract(in_ldef + f$length(cdef) + 1, - + f$length(libdefs) - in_ldef - f$length(cdef) - 1, - + libdefs) +$ else +$ if (f$type('cdef') .eqs. "INTEGER") +$ then +$ write aconf "#define ''cdef' ", 'cdef' +$ else +$ if (f$type('cdef') .eqs. "STRING") +$ then +$ write aconf "#define ''cdef' ", """", '''cdef'', """" +$ else +$ gosub check_cc_def +$ endif +$ endif +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Check if this is a define relating to the properties of the C/C++ +$! compiler +$! +$ CHECK_CC_DEF: +$ if (cdef .eqs. "_LARGEFILE64_SOURCE") +$ then +$ copy sys$input: 'tc' +$ deck +#include "tconfig" +#define _LARGEFILE +#include + +int main(){ +FILE *fp; + fp = fopen("temp.txt","r"); + fseeko(fp,1,SEEK_SET); + fclose(fp); +} + +$ eod +$ test_inv = false +$ comm_h = false +$ gosub cc_prop_check +$ return +$ endif +$ write aconf "/* ", line, " */" +$ return +$!------------------------------------------------------------------------------ +$! +$! Check for properties of C/C++ compiler +$! +$! Version history +$! 0.01 20031020 First version to receive a number +$! 0.02 20031022 Added logic for defines with value +$! 0.03 20040309 Make sure local config file gets not deleted +$! 0.04 20041230 Also write include for configure run +$! 0.05 20050103 Add processing of "comment defines" +$CC_PROP_CHECK: +$ cc_prop = true +$ is_need = false +$ is_need = (f$extract(0,4,cdef) .eqs. "NEED") .or. (test_inv .eq. true) +$ if f$search(th) .eqs. "" then create 'th' +$ set message/nofac/noident/nosever/notext +$ on error then continue +$ cc 'tmpnam' +$ if .not. ($status) then cc_prop = false +$ on error then continue +$! The headers might lie about the capabilities of the RTL +$ link 'tmpnam',tmp.opt/opt +$ if .not. ($status) then cc_prop = false +$ set message/fac/ident/sever/text +$ on error then goto err_exit +$ delete/nolog 'tmpnam'.*;*/exclude='th' +$ if (cc_prop .and. .not. is_need) .or. - + (.not. cc_prop .and. is_need) +$ then +$ write sys$output "Checking for ''cdef'... yes" +$ if f$type('cdef_val'_yes) .nes. "" +$ then +$ if f$type('cdef_val'_yes) .eqs. "INTEGER" - + then call write_config f$fao("#define !AS !UL",cdef,'cdef_val'_yes) +$ if f$type('cdef_val'_yes) .eqs. "STRING" - + then call write_config f$fao("#define !AS !AS",cdef,'cdef_val'_yes) +$ else +$ call write_config f$fao("#define !AS 1",cdef) +$ endif +$ if (cdef .eqs. "HAVE_FSEEKO") .or. (cdef .eqs. "_LARGE_FILES") .or. - + (cdef .eqs. "_LARGEFILE64_SOURCE") then - + call write_config f$string("#define _LARGEFILE 1") +$ else +$ write sys$output "Checking for ''cdef'... no" +$ if (comm_h) +$ then + call write_config f$fao("/* !AS */",line) +$ else +$ if f$type('cdef_val'_no) .nes. "" +$ then +$ if f$type('cdef_val'_no) .eqs. "INTEGER" - + then call write_config f$fao("#define !AS !UL",cdef,'cdef_val'_no) +$ if f$type('cdef_val'_no) .eqs. "STRING" - + then call write_config f$fao("#define !AS !AS",cdef,'cdef_val'_no) +$ else +$ call write_config f$fao("#undef !AS",cdef) +$ endif +$ endif +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Check for properties of C/C++ compiler with multiple result values +$! +$! Version history +$! 0.01 20040127 First version +$! 0.02 20050103 Reconcile changes from cc_prop up to version 0.05 +$CC_MPROP_CHECK: +$ cc_prop = true +$ i = 1 +$ idel = 1 +$ MT_LOOP: +$ if f$type(result_'i') .eqs. "STRING" +$ then +$ set message/nofac/noident/nosever/notext +$ on error then continue +$ cc 'tmpnam'_'i' +$ if .not. ($status) then cc_prop = false +$ on error then continue +$! The headers might lie about the capabilities of the RTL +$ link 'tmpnam'_'i',tmp.opt/opt +$ if .not. ($status) then cc_prop = false +$ set message/fac/ident/sever/text +$ on error then goto err_exit +$ delete/nolog 'tmpnam'_'i'.*;* +$ if (cc_prop) +$ then +$ write sys$output "Checking for ''cdef'... ", mdef_'i' +$ if f$type(mdef_'i') .eqs. "INTEGER" - + then call write_config f$fao("#define !AS !UL",cdef,mdef_'i') +$ if f$type('cdef_val'_yes) .eqs. "STRING" - + then call write_config f$fao("#define !AS !AS",cdef,mdef_'i') +$ goto msym_clean +$ else +$ i = i + 1 +$ goto mt_loop +$ endif +$ endif +$ write sys$output "Checking for ''cdef'... no" +$ call write_config f$fao("#undef !AS",cdef) +$ MSYM_CLEAN: +$ if (idel .le. msym_max) +$ then +$ delete/sym mdef_'idel' +$ idel = idel + 1 +$ goto msym_clean +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Write configuration to both permanent and temporary config file +$! +$! Version history +$! 0.01 20031029 First version to receive a number +$! +$WRITE_CONFIG: SUBROUTINE +$ write aconf 'p1' +$ open/append confh 'th' +$ write confh 'p1' +$ close confh +$ENDSUBROUTINE +$!------------------------------------------------------------------------------ +$! +$! Analyze the project map file and create the symbol vector for a shareable +$! image from it +$! +$! Version history +$! 0.01 20120128 First version +$! 0.02 20120226 Add pre-load logic +$! +$ MAP_2_SHOPT: Subroutine +$! +$ SAY := "WRITE_ SYS$OUTPUT" +$! +$ IF F$SEARCH("''P1'") .EQS. "" +$ THEN +$ SAY "MAP_2_SHOPT-E-NOSUCHFILE: Error, inputfile ''p1' not available" +$ goto exit_m2s +$ ENDIF +$ IF "''P2'" .EQS. "" +$ THEN +$ SAY "MAP_2_SHOPT: Error, no output file provided" +$ goto exit_m2s +$ ENDIF +$! +$ module1 = "deflate#deflateEnd#deflateInit_#deflateParams#deflateSetDictionary" +$ module2 = "gzclose#gzerror#gzgetc#gzgets#gzopen#gzprintf#gzputc#gzputs#gzread" +$ module3 = "gzseek#gztell#inflate#inflateEnd#inflateInit_#inflateSetDictionary" +$ module4 = "inflateSync#uncompress#zlibVersion#compress" +$ open/read map 'p1 +$ if axp .or. ia64 +$ then +$ open/write aopt a.opt +$ open/write bopt b.opt +$ write aopt " CASE_SENSITIVE=YES" +$ write bopt "SYMBOL_VECTOR= (-" +$ mod_sym_num = 1 +$ MOD_SYM_LOOP: +$ if f$type(module'mod_sym_num') .nes. "" +$ then +$ mod_in = 0 +$ MOD_SYM_IN: +$ shared_proc = f$element(mod_in, "#", module'mod_sym_num') +$ if shared_proc .nes. "#" +$ then +$ write aopt f$fao(" symbol_vector=(!AS/!AS=PROCEDURE)",- + f$edit(shared_proc,"upcase"),shared_proc) +$ write bopt f$fao("!AS=PROCEDURE,-",shared_proc) +$ mod_in = mod_in + 1 +$ goto mod_sym_in +$ endif +$ mod_sym_num = mod_sym_num + 1 +$ goto mod_sym_loop +$ endif +$MAP_LOOP: +$ read/end=map_end map line +$ if (f$locate("{",line).lt. f$length(line)) .or. - + (f$locate("global:", line) .lt. f$length(line)) +$ then +$ proc = true +$ goto map_loop +$ endif +$ if f$locate("}",line).lt. f$length(line) then proc = false +$ if f$locate("local:", line) .lt. f$length(line) then proc = false +$ if proc +$ then +$ shared_proc = f$edit(line,"collapse") +$ chop_semi = f$locate(";", shared_proc) +$ if chop_semi .lt. f$length(shared_proc) then - + shared_proc = f$extract(0, chop_semi, shared_proc) +$ write aopt f$fao(" symbol_vector=(!AS/!AS=PROCEDURE)",- + f$edit(shared_proc,"upcase"),shared_proc) +$ write bopt f$fao("!AS=PROCEDURE,-",shared_proc) +$ endif +$ goto map_loop +$MAP_END: +$ close/nolog aopt +$ close/nolog bopt +$ open/append libopt 'p2' +$ open/read aopt a.opt +$ open/read bopt b.opt +$ALOOP: +$ read/end=aloop_end aopt line +$ write libopt line +$ goto aloop +$ALOOP_END: +$ close/nolog aopt +$ sv = "" +$BLOOP: +$ read/end=bloop_end bopt svn +$ if (svn.nes."") +$ then +$ if (sv.nes."") then write libopt sv +$ sv = svn +$ endif +$ goto bloop +$BLOOP_END: +$ write libopt f$extract(0,f$length(sv)-2,sv), "-" +$ write libopt ")" +$ close/nolog bopt +$ delete/nolog/noconf a.opt;*,b.opt;* +$ else +$ if vax +$ then +$ open/append libopt 'p2' +$ mod_sym_num = 1 +$ VMOD_SYM_LOOP: +$ if f$type(module'mod_sym_num') .nes. "" +$ then +$ mod_in = 0 +$ VMOD_SYM_IN: +$ shared_proc = f$element(mod_in, "#", module'mod_sym_num') +$ if shared_proc .nes. "#" +$ then +$ write libopt f$fao("UNIVERSAL=!AS",- + f$edit(shared_proc,"upcase")) +$ mod_in = mod_in + 1 +$ goto vmod_sym_in +$ endif +$ mod_sym_num = mod_sym_num + 1 +$ goto vmod_sym_loop +$ endif +$VMAP_LOOP: +$ read/end=vmap_end map line +$ if (f$locate("{",line).lt. f$length(line)) .or. - + (f$locate("global:", line) .lt. f$length(line)) +$ then +$ proc = true +$ goto vmap_loop +$ endif +$ if f$locate("}",line).lt. f$length(line) then proc = false +$ if f$locate("local:", line) .lt. f$length(line) then proc = false +$ if proc +$ then +$ shared_proc = f$edit(line,"collapse") +$ chop_semi = f$locate(";", shared_proc) +$ if chop_semi .lt. f$length(shared_proc) then - + shared_proc = f$extract(0, chop_semi, shared_proc) +$ write libopt f$fao("UNIVERSAL=!AS",- + f$edit(shared_proc,"upcase")) +$ endif +$ goto vmap_loop +$VMAP_END: +$ else +$ write sys$output "Unknown Architecture (Not VAX, AXP, or IA64)" +$ write sys$output "No options file created" +$ endif +$ endif +$ EXIT_M2S: +$ close/nolog map +$ close/nolog libopt +$ endsubroutine diff --git a/contrib/zlib/treebuild.xml b/contrib/zlib/treebuild.xml new file mode 100644 index 000000000..6bd6677c4 --- /dev/null +++ b/contrib/zlib/treebuild.xml @@ -0,0 +1,116 @@ + + + + zip compression library + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/zlib/trees.c b/contrib/zlib/trees.c index 1fd7759ef..50cf4b457 100644 --- a/contrib/zlib/trees.c +++ b/contrib/zlib/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2012 Jean-loup Gailly + * Copyright (C) 1995-2017 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -36,7 +36,7 @@ #include "deflate.h" -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include #endif @@ -122,13 +122,13 @@ struct static_tree_desc_s { int max_length; /* max bit length for the codes */ }; -local static_tree_desc static_l_desc = +local const static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; -local static_tree_desc static_d_desc = +local const static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; -local static_tree_desc static_bl_desc = +local const static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== @@ -152,18 +152,16 @@ local int detect_data_type OF((deflate_state *s)); local unsigned bi_reverse OF((unsigned value, int length)); local void bi_windup OF((deflate_state *s)); local void bi_flush OF((deflate_state *s)); -local void copy_block OF((deflate_state *s, charf *buf, unsigned len, - int header)); #ifdef GEN_TREES_H local void gen_trees_header OF((void)); #endif -#ifndef DEBUG +#ifndef ZLIB_DEBUG # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) /* Send a code of the given tree. c and tree must not have side effects */ -#else /* DEBUG */ +#else /* !ZLIB_DEBUG */ # define send_code(s, c, tree) \ { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ send_bits(s, tree[c].Code, tree[c].Len); } @@ -182,7 +180,7 @@ local void gen_trees_header OF((void)); * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void send_bits OF((deflate_state *s, int value, int length)); local void send_bits(s, value, length) @@ -208,12 +206,12 @@ local void send_bits(s, value, length) s->bi_valid += length; } } -#else /* !DEBUG */ +#else /* !ZLIB_DEBUG */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ + int val = (int)value;\ s->bi_buf |= (ush)val << s->bi_valid;\ put_short(s, s->bi_buf);\ s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ @@ -223,7 +221,7 @@ local void send_bits(s, value, length) s->bi_valid += len;\ }\ } -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* the arguments must not have side effects */ @@ -317,7 +315,7 @@ local void tr_static_init() * Genererate the file trees.h describing the static trees. */ #ifdef GEN_TREES_H -# ifndef DEBUG +# ifndef ZLIB_DEBUG # include # endif @@ -394,7 +392,7 @@ void ZLIB_INTERNAL _tr_init(s) s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len = 0L; s->bits_sent = 0L; #endif @@ -522,12 +520,12 @@ local void gen_bitlen(s, desc) xbits = 0; if (n >= base) xbits = extra[n-base]; f = tree[n].Freq; - s->opt_len += (ulg)f * (bits + xbits); - if (stree) s->static_len += (ulg)f * (stree[n].Len + xbits); + s->opt_len += (ulg)f * (unsigned)(bits + xbits); + if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits); } if (overflow == 0) return; - Trace((stderr,"\nbit length overflow\n")); + Tracev((stderr,"\nbit length overflow\n")); /* This happens for example on obj2 and pic of the Calgary corpus */ /* Find the first bit length which could increase: */ @@ -554,9 +552,8 @@ local void gen_bitlen(s, desc) m = s->heap[--h]; if (m > max_code) continue; if ((unsigned) tree[m].Len != (unsigned) bits) { - Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); - s->opt_len += ((long)bits - (long)tree[m].Len) - *(long)tree[m].Freq; + Tracev((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + s->opt_len += ((ulg)bits - tree[m].Len) * tree[m].Freq; tree[m].Len = (ush)bits; } n--; @@ -578,7 +575,7 @@ local void gen_codes (tree, max_code, bl_count) ushf *bl_count; /* number of codes at each bit length */ { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - ush code = 0; /* running code value */ + unsigned code = 0; /* running code value */ int bits; /* bit index */ int n; /* code index */ @@ -586,7 +583,8 @@ local void gen_codes (tree, max_code, bl_count) * without bit reversal. */ for (bits = 1; bits <= MAX_BITS; bits++) { - next_code[bits] = code = (code + bl_count[bits-1]) << 1; + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = (ush)code; } /* Check that the bit counts in bl_count are consistent. The last code * must be all ones. @@ -599,7 +597,7 @@ local void gen_codes (tree, max_code, bl_count) int len = tree[n].Len; if (len == 0) continue; /* Now reverse the bits */ - tree[n].Code = bi_reverse(next_code[len]++, len); + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); @@ -821,7 +819,7 @@ local int build_bl_tree(s) if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; } /* Update opt_len to include the bit length tree and counts */ - s->opt_len += 3*(max_blindex+1) + 5+5+4; + s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4; Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", s->opt_len, s->static_len)); @@ -869,11 +867,17 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) int last; /* one if this is the last block for a file */ { send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ -#ifdef DEBUG + bi_windup(s); /* align on byte boundary */ + put_short(s, (ush)stored_len); + put_short(s, (ush)~stored_len); + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + s->pending += stored_len; +#ifdef ZLIB_DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; s->compressed_len += (stored_len + 4) << 3; + s->bits_sent += 2*16; + s->bits_sent += stored_len<<3; #endif - copy_block(s, buf, (unsigned)stored_len, 1); /* with header */ } /* =========================================================================== @@ -894,7 +898,7 @@ void ZLIB_INTERNAL _tr_align(s) { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ #endif bi_flush(s); @@ -902,7 +906,7 @@ void ZLIB_INTERNAL _tr_align(s) /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static - * trees or store, and output the encoded block to the zip file. + * trees or store, and write out the encoded block. */ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) deflate_state *s; @@ -974,7 +978,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) send_bits(s, (STATIC_TREES<<1)+last, 3); compress_block(s, (const ct_data *)static_ltree, (const ct_data *)static_dtree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->static_len; #endif } else { @@ -983,7 +987,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) max_blindex+1); compress_block(s, (const ct_data *)s->dyn_ltree, (const ct_data *)s->dyn_dtree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->opt_len; #endif } @@ -995,7 +999,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) if (last) { bi_windup(s); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 7; /* align on byte boundary */ #endif } @@ -1090,7 +1094,7 @@ local void compress_block(s, ltree, dtree) send_code(s, code, dtree); /* send the distance code */ extra = extra_dbits[code]; if (extra != 0) { - dist -= base_dist[code]; + dist -= (unsigned)base_dist[code]; send_bits(s, dist, extra); /* send the extra distance bits */ } } /* literal or match pair ? */ @@ -1193,34 +1197,7 @@ local void bi_windup(s) } s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->bits_sent = (s->bits_sent+7) & ~7; #endif } - -/* =========================================================================== - * Copy a stored block, storing first the length and its - * one's complement if requested. - */ -local void copy_block(s, buf, len, header) - deflate_state *s; - charf *buf; /* the input data */ - unsigned len; /* its length */ - int header; /* true if block header must be written */ -{ - bi_windup(s); /* align on byte boundary */ - - if (header) { - put_short(s, (ush)len); - put_short(s, (ush)~len); -#ifdef DEBUG - s->bits_sent += 2*16; -#endif - } -#ifdef DEBUG - s->bits_sent += (ulg)len<<3; -#endif - while (len--) { - put_byte(s, *buf++); - } -} diff --git a/contrib/zlib/uncompr.c b/contrib/zlib/uncompr.c index 242e9493d..f03a1a865 100644 --- a/contrib/zlib/uncompr.c +++ b/contrib/zlib/uncompr.c @@ -1,5 +1,5 @@ /* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003, 2010 Jean-loup Gailly. + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,51 +9,85 @@ #include "zlib.h" /* =========================================================================== - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. */ +int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong *sourceLen; +{ + z_stream stream; + int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ + + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; + + err = inflateInit(&stream); + if (err != Z_OK) return err; + + stream.next_out = dest; + stream.avail_out = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} + int ZEXPORT uncompress (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; uLong sourceLen; { - z_stream stream; - int err; - - stream.next_in = (z_const Bytef *)source; - stream.avail_in = (uInt)sourceLen; - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; - - stream.zalloc = (alloc_func)0; - stream.zfree = (free_func)0; - - err = inflateInit(&stream); - if (err != Z_OK) return err; - - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; - return err; - } - *destLen = stream.total_out; - - err = inflateEnd(&stream); - return err; + return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/contrib/zlib/win32/Makefile.gcc b/contrib/zlib/win32/Makefile.gcc index 6d1ded622..305be50af 100644 --- a/contrib/zlib/win32/Makefile.gcc +++ b/contrib/zlib/win32/Makefile.gcc @@ -39,7 +39,7 @@ IMPLIB = libz.dll.a SHARED_MODE=0 #LOC = -DASMV -#LOC = -DDEBUG -g +#LOC = -DZLIB_DEBUG -g PREFIX = CC = $(PREFIX)gcc diff --git a/contrib/zlib/win32/Makefile.msc b/contrib/zlib/win32/Makefile.msc index 67b773171..6831882de 100644 --- a/contrib/zlib/win32/Makefile.msc +++ b/contrib/zlib/win32/Makefile.msc @@ -1,5 +1,5 @@ # Makefile for zlib using Microsoft (Visual) C -# zlib is copyright (C) 1995-2006 Jean-loup Gailly and Mark Adler +# zlib is copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler # # Usage: # nmake -f win32/Makefile.msc (standard build) diff --git a/contrib/zlib/win32/README-WIN32.txt b/contrib/zlib/win32/README-WIN32.txt index 3d77d521e..a1de359a3 100644 --- a/contrib/zlib/win32/README-WIN32.txt +++ b/contrib/zlib/win32/README-WIN32.txt @@ -1,6 +1,6 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.2.8 is a general purpose data compression library. All the code is +zlib 1.2.11.1 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) @@ -22,7 +22,7 @@ before asking for help. Manifest: -The package zlib-1.2.8-win32-x86.zip will contain the following files: +The package zlib-1.2.11.1-win32-x86.zip will contain the following files: README-WIN32.txt This document ChangeLog Changes since previous zlib packages @@ -72,7 +72,7 @@ are too numerous to cite here. Copyright notice: - (C) 1995-2012 Jean-loup Gailly and Mark Adler + (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/contrib/zlib/win32/VisualC.txt b/contrib/zlib/win32/VisualC.txt index 579a5fc9e..1005b2194 100644 --- a/contrib/zlib/win32/VisualC.txt +++ b/contrib/zlib/win32/VisualC.txt @@ -1,3 +1,3 @@ To build zlib using the Microsoft Visual C++ environment, -use the appropriate project from the projects/ directory. +use the appropriate project from the contrib/vstudio/ directory. diff --git a/contrib/zlib/win32/zlib.def b/contrib/zlib/win32/zlib.def index face65518..a2188b000 100644 --- a/contrib/zlib/win32/zlib.def +++ b/contrib/zlib/win32/zlib.def @@ -8,6 +8,7 @@ EXPORTS inflateEnd ; advanced functions deflateSetDictionary + deflateGetDictionary deflateCopy deflateReset deflateParams @@ -33,12 +34,15 @@ EXPORTS compress2 compressBound uncompress + uncompress2 gzopen gzdopen gzbuffer gzsetparams gzread + gzfread gzwrite + gzfwrite gzprintf gzvprintf gzputs @@ -67,7 +71,9 @@ EXPORTS crc32_combine64 ; checksum functions adler32 + adler32_z crc32 + crc32_z adler32_combine crc32_combine ; various hacks, don't look :) @@ -81,6 +87,8 @@ EXPORTS inflateSyncPoint get_crc_table inflateUndermine + inflateValidate + inflateCodesUsed inflateResetKeep deflateResetKeep gzopen_w diff --git a/contrib/zlib/win32/zlib1.rc b/contrib/zlib/win32/zlib1.rc index 5c0feed1b..234e641c3 100644 --- a/contrib/zlib/win32/zlib1.rc +++ b/contrib/zlib/win32/zlib1.rc @@ -26,7 +26,7 @@ BEGIN VALUE "FileDescription", "zlib data compression library\0" VALUE "FileVersion", ZLIB_VERSION "\0" VALUE "InternalName", "zlib1.dll\0" - VALUE "LegalCopyright", "(C) 1995-2013 Jean-loup Gailly & Mark Adler\0" + VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0" VALUE "OriginalFilename", "zlib1.dll\0" VALUE "ProductName", "zlib\0" VALUE "ProductVersion", ZLIB_VERSION "\0" diff --git a/contrib/zlib/zconf.h.cmakein b/contrib/zlib/zconf.h.cmakein index 043019cda..a7f24cce6 100644 --- a/contrib/zlib/zconf.h.cmakein +++ b/contrib/zlib/zconf.h.cmakein @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -19,7 +19,7 @@ #ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ # define Z_PREFIX_SET -/* all linked symbols */ +/* all linked symbols and init macros */ # define _dist_code z__dist_code # define _length_code z__length_code # define _tr_align z__tr_align @@ -31,6 +31,7 @@ # define adler32 z_adler32 # define adler32_combine z_adler32_combine # define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z # ifndef Z_SOLO # define compress z_compress # define compress2 z_compress2 @@ -39,10 +40,14 @@ # define crc32 z_crc32 # define crc32_combine z_crc32_combine # define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate # define deflateBound z_deflateBound # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 # define deflateInit2_ z_deflateInit2_ # define deflateInit_ z_deflateInit_ # define deflateParams z_deflateParams @@ -69,6 +74,8 @@ # define gzeof z_gzeof # define gzerror z_gzerror # define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite # define gzgetc z_gzgetc # define gzgetc_ z_gzgetc_ # define gzgets z_gzgets @@ -80,7 +87,6 @@ # define gzopen_w z_gzopen_w # endif # define gzprintf z_gzprintf -# define gzvprintf z_gzvprintf # define gzputc z_gzputc # define gzputs z_gzputs # define gzread z_gzread @@ -91,32 +97,39 @@ # define gztell z_gztell # define gztell64 z_gztell64 # define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf # define gzwrite z_gzwrite # endif # define inflate z_inflate # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit # define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed # define inflateCopy z_inflateCopy # define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary # define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ # define inflateInit_ z_inflateInit_ # define inflateMark z_inflateMark # define inflatePrime z_inflatePrime # define inflateReset z_inflateReset # define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary -# define inflateGetDictionary z_inflateGetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateUndermine z_inflateUndermine -# define inflateResetKeep z_inflateResetKeep +# define inflateValidate z_inflateValidate # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table # ifndef Z_SOLO # define uncompress z_uncompress +# define uncompress2 z_uncompress2 # endif # define zError z_zError # ifndef Z_SOLO @@ -226,9 +239,19 @@ # define z_const #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -258,7 +281,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ diff --git a/contrib/zlib/zconf.in.h b/contrib/zlib/zconf.h.in similarity index 50% rename from contrib/zlib/zconf.in.h rename to contrib/zlib/zconf.h.in index 03a9431c8..5e1d68a00 100644 --- a/contrib/zlib/zconf.in.h +++ b/contrib/zlib/zconf.h.in @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -11,52 +11,158 @@ /* * If you *really* need a unique prefix for all types and library functions, * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + * Even better than compiling with -DZ_PREFIX would be to use configure to set + * this permanently in zconf.h using "./configure --zprefix". */ -#ifdef Z_PREFIX -# define deflateInit_ z_deflateInit_ +#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET + +/* all linked symbols and init macros */ +# define _dist_code z__dist_code +# define _length_code z__length_code +# define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits +# define _tr_flush_block z__tr_flush_block +# define _tr_init z__tr_init +# define _tr_stored_block z__tr_stored_block +# define _tr_tally z__tr_tally +# define adler32 z_adler32 +# define adler32_combine z_adler32_combine +# define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif +# define crc32 z_crc32 +# define crc32_combine z_crc32_combine +# define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate -# define deflateEnd z_deflateEnd -# define inflateInit_ z_inflateInit_ -# define inflate z_inflate -# define inflateEnd z_inflateEnd -# define deflateInit2_ z_deflateInit2_ -# define deflateSetDictionary z_deflateSetDictionary -# define deflateCopy z_deflateCopy -# define deflateReset z_deflateReset -# define deflateParams z_deflateParams # define deflateBound z_deflateBound +# define deflateCopy z_deflateCopy +# define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 +# define deflateInit2_ z_deflateInit2_ +# define deflateInit_ z_deflateInit_ +# define deflateParams z_deflateParams +# define deflatePending z_deflatePending # define deflatePrime z_deflatePrime +# define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep +# define deflateSetDictionary z_deflateSetDictionary +# define deflateSetHeader z_deflateSetHeader +# define deflateTune z_deflateTune +# define deflate_copyright z_deflate_copyright +# define get_crc_table z_get_crc_table +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif +# define inflate z_inflate +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit +# define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed +# define inflateCopy z_inflateCopy +# define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary +# define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ +# define inflateInit_ z_inflateInit_ +# define inflateMark z_inflateMark +# define inflatePrime z_inflatePrime +# define inflateReset z_inflateReset +# define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint -# define inflateCopy z_inflateCopy -# define inflateReset z_inflateReset -# define inflateBack z_inflateBack -# define inflateBackEnd z_inflateBackEnd -# define compress z_compress -# define compress2 z_compress2 -# define compressBound z_compressBound -# define uncompress z_uncompress -# define adler32 z_adler32 -# define crc32 z_crc32 -# define get_crc_table z_get_crc_table +# define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate +# define inflate_copyright z_inflate_copyright +# define inflate_fast z_inflate_fast +# define inflate_table z_inflate_table +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif # define zError z_zError +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif +# define zlibCompileFlags z_zlibCompileFlags +# define zlibVersion z_zlibVersion -# define alloc_func z_alloc_func -# define free_func z_free_func -# define in_func z_in_func -# define out_func z_out_func +/* all zlib typedefs in zlib.h and zconf.h */ # define Byte z_Byte -# define uInt z_uInt -# define uLong z_uLong # define Bytef z_Bytef +# define alloc_func z_alloc_func # define charf z_charf +# define free_func z_free_func +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif +# define gz_header z_gz_header +# define gz_headerp z_gz_headerp +# define in_func z_in_func # define intf z_intf +# define out_func z_out_func +# define uInt z_uInt # define uIntf z_uIntf +# define uLong z_uLong # define uLongf z_uLongf -# define voidpf z_voidpf # define voidp z_voidp +# define voidpc z_voidpc +# define voidpf z_voidpf + +/* all zlib structs in zlib.h and zconf.h */ +# define gz_header_s z_gz_header_s +# define internal_state z_internal_state + #endif #if defined(__MSDOS__) && !defined(MSDOS) @@ -125,9 +231,25 @@ # endif #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -157,7 +279,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ @@ -171,6 +293,14 @@ # endif #endif +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -284,49 +414,121 @@ typedef uLong FAR uLongf; typedef Byte *voidp; #endif -#if 0 /* HAVE_UNISTD_H -- this line is updated by ./configure */ -# include /* for off_t */ -# include /* for SEEK_* and off_t */ -# ifdef VMS -# include /* for off_t */ +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short # endif -# define z_off_t off_t #endif -#ifndef SEEK_SET + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + +#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + +#ifdef STDC +# ifndef Z_SOLO +# include /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include /* for wchar_t */ +# endif +#endif + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) +# define Z_HAVE_UNISTD_H +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) # define SEEK_SET 0 /* Seek from beginning of file. */ # define SEEK_CUR 1 /* Seek from current position. */ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ #endif + #ifndef z_off_t # define z_off_t long #endif -#if defined(__OS400__) -# define NO_vsnprintf -#endif - -#if defined(__MVS__) -# define NO_vsnprintf -# ifdef FAR -# undef FAR +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t # endif #endif /* MVS linker does not support external names larger than 8 bytes */ #if defined(__MVS__) -# pragma map(deflateInit_,"DEIN") -# pragma map(deflateInit2_,"DEIN2") -# pragma map(deflateEnd,"DEEND") -# pragma map(deflateBound,"DEBND") -# pragma map(inflateInit_,"ININ") -# pragma map(inflateInit2_,"ININ2") -# pragma map(inflateEnd,"INEND") -# pragma map(inflateSync,"INSY") -# pragma map(inflateSetDictionary,"INSEDI") -# pragma map(compressBound,"CMBND") -# pragma map(inflate_table,"INTABL") -# pragma map(inflate_fast,"INFA") -# pragma map(inflate_copyright,"INCOPY") + #pragma map(deflateInit_,"DEIN") + #pragma map(deflateInit2_,"DEIN2") + #pragma map(deflateEnd,"DEEND") + #pragma map(deflateBound,"DEBND") + #pragma map(inflateInit_,"ININ") + #pragma map(inflateInit2_,"ININ2") + #pragma map(inflateEnd,"INEND") + #pragma map(inflateSync,"INSY") + #pragma map(inflateSetDictionary,"INSEDI") + #pragma map(compressBound,"CMBND") + #pragma map(inflate_table,"INTABL") + #pragma map(inflate_fast,"INFA") + #pragma map(inflate_copyright,"INCOPY") #endif #endif /* ZCONF_H */ diff --git a/contrib/zlib/zconf.h.included b/contrib/zlib/zconf.h.included index 9ccbb0a6f..5e1d68a00 100644 --- a/contrib/zlib/zconf.h.included +++ b/contrib/zlib/zconf.h.included @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -7,8 +7,6 @@ #ifndef ZCONF_H #define ZCONF_H -/* #undef Z_PREFIX */ -/* #undef Z_HAVE_UNISTD_H */ /* * If you *really* need a unique prefix for all types and library functions, @@ -19,7 +17,7 @@ #ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ # define Z_PREFIX_SET -/* all linked symbols */ +/* all linked symbols and init macros */ # define _dist_code z__dist_code # define _length_code z__length_code # define _tr_align z__tr_align @@ -31,6 +29,7 @@ # define adler32 z_adler32 # define adler32_combine z_adler32_combine # define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z # ifndef Z_SOLO # define compress z_compress # define compress2 z_compress2 @@ -39,10 +38,14 @@ # define crc32 z_crc32 # define crc32_combine z_crc32_combine # define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate # define deflateBound z_deflateBound # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 # define deflateInit2_ z_deflateInit2_ # define deflateInit_ z_deflateInit_ # define deflateParams z_deflateParams @@ -69,6 +72,8 @@ # define gzeof z_gzeof # define gzerror z_gzerror # define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite # define gzgetc z_gzgetc # define gzgetc_ z_gzgetc_ # define gzgets z_gzgets @@ -80,7 +85,6 @@ # define gzopen_w z_gzopen_w # endif # define gzprintf z_gzprintf -# define gzvprintf z_gzvprintf # define gzputc z_gzputc # define gzputs z_gzputs # define gzread z_gzread @@ -91,32 +95,39 @@ # define gztell z_gztell # define gztell64 z_gztell64 # define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf # define gzwrite z_gzwrite # endif # define inflate z_inflate # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit # define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed # define inflateCopy z_inflateCopy # define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary # define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ # define inflateInit_ z_inflateInit_ # define inflateMark z_inflateMark # define inflatePrime z_inflatePrime # define inflateReset z_inflateReset # define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary -# define inflateGetDictionary z_inflateGetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateUndermine z_inflateUndermine -# define inflateResetKeep z_inflateResetKeep +# define inflateValidate z_inflateValidate # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table # ifndef Z_SOLO # define uncompress z_uncompress +# define uncompress2 z_uncompress2 # endif # define zError z_zError # ifndef Z_SOLO @@ -226,9 +237,19 @@ # define z_const #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -258,7 +279,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ diff --git a/contrib/zlib/zlib.3 b/contrib/zlib/zlib.3 new file mode 100644 index 000000000..fac206b52 --- /dev/null +++ b/contrib/zlib/zlib.3 @@ -0,0 +1,149 @@ +.TH ZLIB 3 "xx Jan 2017" +.SH NAME +zlib \- compression/decompression library +.SH SYNOPSIS +[see +.I zlib.h +for full description] +.SH DESCRIPTION +The +.I zlib +library is a general purpose data compression library. +The code is thread safe, assuming that the standard library functions +used are thread safe, such as memory allocation routines. +It provides in-memory compression and decompression functions, +including integrity checks of the uncompressed data. +This version of the library supports only one compression method (deflation) +but other algorithms may be added later +with the same stream interface. +.LP +Compression can be done in a single step if the buffers are large enough +or can be done by repeated calls of the compression function. +In the latter case, +the application must provide more input and/or consume the output +(providing more output space) before each call. +.LP +The library also supports reading and writing files in +.IR gzip (1) +(.gz) format +with an interface similar to that of stdio. +.LP +The library does not install any signal handler. +The decoder checks the consistency of the compressed data, +so the library should never crash even in the case of corrupted input. +.LP +All functions of the compression library are documented in the file +.IR zlib.h . +The distribution source includes examples of use of the library +in the files +.I test/example.c +and +.IR test/minigzip.c, +as well as other examples in the +.IR examples/ +directory. +.LP +Changes to this version are documented in the file +.I ChangeLog +that accompanies the source. +.LP +.I zlib +is built in to many languages and operating systems, including but not limited to +Java, Python, .NET, PHP, Perl, Ruby, Swift, and Go. +.LP +An experimental package to read and write files in the .zip format, +written on top of +.I zlib +by Gilles Vollant (info@winimage.com), +is available at: +.IP +http://www.winimage.com/zLibDll/minizip.html +and also in the +.I contrib/minizip +directory of the main +.I zlib +source distribution. +.SH "SEE ALSO" +The +.I zlib +web site can be found at: +.IP +http://zlib.net/ +.LP +The data format used by the +.I zlib +library is described by RFC +(Request for Comments) 1950 to 1952 in the files: +.IP +http://tools.ietf.org/html/rfc1950 (for the zlib header and trailer format) +.br +http://tools.ietf.org/html/rfc1951 (for the deflate compressed data format) +.br +http://tools.ietf.org/html/rfc1952 (for the gzip header and trailer format) +.LP +Mark Nelson wrote an article about +.I zlib +for the Jan. 1997 issue of Dr. Dobb's Journal; +a copy of the article is available at: +.IP +http://marknelson.us/1997/01/01/zlib-engine/ +.SH "REPORTING PROBLEMS" +Before reporting a problem, +please check the +.I zlib +web site to verify that you have the latest version of +.IR zlib ; +otherwise, +obtain the latest version and see if the problem still exists. +Please read the +.I zlib +FAQ at: +.IP +http://zlib.net/zlib_faq.html +.LP +before asking for help. +Send questions and/or comments to zlib@gzip.org, +or (for the Windows DLL version) to Gilles Vollant (info@winimage.com). +.SH AUTHORS AND LICENSE +Version 1.2.11.1 +.LP +Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler +.LP +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. +.LP +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: +.LP +.nr step 1 1 +.IP \n[step]. 3 +The origin of this software must not be misrepresented; you must not +claim that you wrote the original software. If you use this software +in a product, an acknowledgment in the product documentation would be +appreciated but is not required. +.IP \n+[step]. +Altered source versions must be plainly marked as such, and must not be +misrepresented as being the original software. +.IP \n+[step]. +This notice may not be removed or altered from any source distribution. +.LP +Jean-loup Gailly Mark Adler +.br +jloup@gzip.org madler@alumni.caltech.edu +.LP +The deflate format used by +.I zlib +was defined by Phil Katz. +The deflate and +.I zlib +specifications were written by L. Peter Deutsch. +Thanks to all the people who reported problems and suggested various +improvements in +.IR zlib ; +who are too numerous to cite here. +.LP +UNIX manual page by R. P. C. Rodgers, +U.S. National Library of Medicine (rodgers@nlm.nih.gov). +.\" end of man page diff --git a/contrib/zlib/zlib.3.pdf b/contrib/zlib/zlib.3.pdf new file mode 100644 index 0000000000000000000000000000000000000000..48e6147385dc9dfc0058474f2e1efa1453409e8c GIT binary patch literal 19320 zcmch<1yo$ivH%)f1Hm<9u;2`XJ0Z9QcbCDP!2^WgmIQZq3GNbt1$Wn=!6lHO0scU8 z&dELJ-uLeR>#aYtW~O&{S9NtsckSxhL#-ex&IDp+N2TuGdz*#I0bl{x8Cjz8^8;nU zwq_7>04KCY2`Fx14FNj>fa2DM5U?oN*v@ zq<8QqT8NH6&obFpvhGp>TTM&b_)<|B$-%P2XvMV}o++O`Is3N2@`_fqQu{?L7$#wR zThQwgA}?6=4h?_o$;p|cr{gKPvFoB#&%q19bmZ^v5?kZ6PmhU#>)*n5=W(th3=&N= zNvn4ErBv4W+vLw^+CBr9nwwFwl8Kyz21{>4$`Ianhxw}oE4g?%U^jby?Z4RLvjhL| zW~OS=Ys1jmo6plLzbywhE$$bW7SDgNR&~EMa{G|KA*OZ2?x%VVVRla`9_|__XdD;{@jMf`9_^1tBRXNo`T%hBqex z3T6i*>W(g^*^g3}v=mkU*I~^^7+04+b|!9ZYoZ)6zZURj+HOmVAurrh9sb^bf3PK{whk~ zIm~0db)Mm7G)-okPj^vLDup?Qi=1gE|=v>NI&R;nPvsj@TG1|!`<3V*^d{|^jQ z@-2rH*SgMqeTLgHG#E_6(~oPNo$~QRLKDmq{RDtx&%Kt)`pHPtN_P8ARS(oLcE$YT zSueu%!wMcv5jQREXKG?g^SW#m;W3;|DtBW=u1lF@(bcn~E%T@Pq!}lZ##yPkSZmv8 zgDpizKkizzNAp*8`_OvDOOPRl$}&~WR`*ARgRf3UrsZ-u1JP7pfKi5VNLsi$l%j$} zG(4SiH1Kso+gCj5)ZHbT=9sFe6g|BQBUMUlzodPY%6Sju&RQ?g$IvcP5C86qBql?j|x5ZW)){f%?kV-7D9?*f>_2j5?%z0L1w2Xh2E%_G8?x zB@Ye)@R$li(7B|A+T0>9bnIQX{p+t-u%%z&{UWYkq2-dQFN?fJK{N4C->PoWqZIlu zU=l81f~Df=Y}xWQKAk)Re6fcNabm^+0ryjd<9*b$95S(e(&yAxX*~lQBUM@mBRd5nK{A&&SD_Uoq1X^^ zL`FOSAEYgY0$5%^bD>Zkgui<6wL5R&vucUuT*qP(`Li26w8!$jV&=-nZ8=ma2twTL zGX91)UNyzb@9?H(tr^83rH_MFI0&58v=b>}L$sE>$d>trUk zhT3$V!dBnbpO!OV*OyaJ^Srmv-*O4&C$DNu_3fXq`c_ZhR=rO15HZ3RjaQ;*dv7jp z(UnHA^Mu8{6a$m$3;ua<(R__;{*pCWke0=_$Jt53rr6z!Zk$UYt z*={E{-|+(XrDSvEXe?~LntiN}X{1cm$tn6>Sd{~wHBphvSnFuzT~7d3N}qe z+UX?b7PW&1es?BpZB|ZYSGFWP9%gE+oVuS#(uQArfm1Z98xKlVMa&rkN{lEUE}|HF zfuB4>QVe|H!}rW6yQk%v6?v?EVnvpO68y67laN2CeE1|zj44ey1AM=(IGhpLVwNj` zJ)6;5K}T}98M0)?c}2+1Aq{JyNroVI7z+=xXpT_n$n%xp!fEzE&J=5$)TJ{tR0jR5 zpIc;*to{k!f@D6yt0BPmH}(Zi?yuj_c_}f#zPavXDE$*T74Q6FS2f32^!xYU%%H#E-zpEUL5C>E6>1A~myyQiJjiKe{} zllmfhLbOSwkRJCzc7}>$mglp`Sr>8J~zPzfNx2;)ZCguEVe}z#;mB8>HFGshOP+U!4=wN*E7R%9;qf??cf3TUEIuHM@ z84(h|7o>e*e1zhb`LS>|0QkI@MRZbHA(oW%2O>=2IdSTPz>r+T0jW! zR&@0P#`ORv8`UH2UAU)d;?7lcB$h%u;%9iG2K#T%3XrwyT>&@>gM)i9ruw=4IBZmX z?t3w6+ccqNFEh#rY>!^mR_(QNjYO^;;2q~5x_(YiUHtrc7+aa_+~8dZkO*?RpQYEl zWUT^|eVLn`Oo?QZPmJHFys_)uqzF)445wpTBg7j)A6g~fQK%PyZMAO-N5=Wi$v37P z(J#$d*ib`P(gsuctRgYI0i|%h(l&2PxF<+faAn8cScuB|BrHCY`-)sNQqTcQb@dgaX;y;-$$ve7v4A*JcqR>6kf#>uFS14pg zd~d~1U$Q)A*V@|yq3A-vOYoTsD_hSqAj)uyWt|EFZ}ml%8sg?Ol zrHM)@Ol=j&$wd;?(!Qa5{@HVIkiq!z51|CQH;GbCgWg+*vYs{({gb&1RiC!iZ|v6A zL2Ctd3Fiv_IA@u`qQhE*qq(x|{%(LR{Rb=Z^!wOpqwhYe3{$Z&EjFR1e11k&ZLcWR z_mWc`_Nnaa4|+-*TGc@}qXqO?-=n%!@CAxyDJt8z2s{8qqNEL@BudS%Op(K8b}v3# zdW(;%a*}F3xYlxA^d)}5#wLNt=kU!#Xf9~Ix(Ur)z(|%$QewVFL0MLMt;_Xkz&>n4 z3o1%uU3cplh0{z`-og(SR`TcM!~RwR5_OaT$rDKXwwCqzN>Ao!J~RrrL>Svk6IOw8 zkMoQ#HHvbcFJ|4KyoUv?Tg4(*o)f~#Wm%&HN5GN@LCkR5suV_i;M3eE7bfMvb7P8= zgX`I+#~l|NSD203Qm=Qp4!up|57H0y^-^vfIV=WRqJC61a>Q#Vy5`1dN#|F5{Xm@8 zfaM$wuFIKj!l?m5c0DzeXz!(&eWq*_`^FB8%F48tkIvU^lXc zEp!F55u~@uVG*QT($7@xam19g#=NovR+3cHduHsi0 zmDgD#s}xxE1uPC6BR)NYd%dogTG_hqn1cJs>0Tiv--mRqUxBB_5`y^Nei!=DS>NFPiu#;giE^(PuH*Vm@)|>9@|I zzQKl%C2z|ju8>-lE(Z7wm&to~rQgCjMtSytZB2gN9YRZYca(Sck$1mroGk39zv@6- z|G3=*asM~>o6Z{ZQ4`$QUS(w?Brc-yQ&BG$-7|y8T?{=m@Ixaa0=(>-CMFT}QT1s< z?Gmmx1?V(ui(hR8vDU`)@w*?Jgbp(1Gz{isw27tccTZ_q>R%~f{<^U-+V6v+C>Y_RX z8z2W7?rx;!0kZYdsW!VuNQXqL1e*9ZU;y^1(&k{TWjE!Oww~H#;>KXwb)>5)pS&z7 z1&sgb`bz$M*Y|fp2j4%8nGUNJ#RNHiRWm5|y=|%UWb~Y>!|dqoo?=u^jCW01*{HDc z${;~r@uEBP7*1QiuH8#tC|Bm;uU&gk_v7|J&-Qs~467OC`$W|I@nMgBl^D9Kw4vIJ zmjW)0hni^CJI1Q3{y6(Ruw+ygrs}OhT&ZurAqmHMxU;1UMxrPgzEL@cUCyJtKw~XG zhx>uEfzA=}guV+*?D?Kk?IQ}2oZ2cNE;<&-&<$5Ax-7^J_q>drlRpVhJNBtgXb*_C zJopRc$hL<|*=%;{X$k#_r@j(=$T9sO*~>KuG8Zq>N{VU-P5g%w@2?`{PbC;g6*rL< z*;tHZq-3&37_KUL%7aSZ_X=>HCe>iN=~R*^8f^zPEz9j`XBv?`I1K17V|swYz3fp* zALyY=J&TT^CKEkb$+hv?2~&9yMT<00X9&wpS5R9D%Oft3tuv!-1e@8VDut8gYs!X` zXJqT<$ z^eHEPpXCT`MO%Sn{ux+Ewwau?&r@IaUASK!7&CQh@**lu@wG~m$Q8$rQPi8SH`&aH zywSAvtL~~bA1HT1vLse8t&Z`Zj^@t~NBeY&N;YR_j#=hBlaAx{01}SaQ)pacy(k~G0#fFx?q(!8S@d#pMbz9O5$#%wC^YOB zIKOqOt!z%W3`wXSv3Y)_r;MfKqE*x$q7~=OFqt#fyhkMDucoLOCyqBFy?XvUfd5<_ zc8piQU!%Z{rLN&ix6M$MuXCfVE|BG#R@RMFw|Yl2Z)%bFy!L~v(hdh&E!I&=gXM)m za-I$MkdiDbAD^Qh9oc*P08s={h(i>yrV+2yLn?06Q`s4*cg)&v!>5hYh`e#1(wg~T zA))*1MLV#pwj=oP4ZA9MS06Rly`;*P$NVfk-rbIlQz;VQX2XYN*92GjoHOmqqy`r? zus{{}tft<>ZSf|h4gww+pg&%6ur zEMUZU07;(wE$v&td_#eb2}#erX**T-!@#hurb03%GqvoO#K>!%5B+7%_S8@CW|*V9 z8alc2Z>>{Yt58_@!=l}l+*%So;^syr61@d?c@GEKEqs1jPG)A+qtF$IQ~Us-QI{&a z_<;|;x{p8EO+ymqLPAXd8tpfKwH2mNle z?L74-k5%yr3Xjfr!*)=kMhvqp_Ri(rieO4d5`Y955}7fOT@iYd-qzadhhZC?=bhKbL#}5L!C?#z<@=Uq**W-T; zOE%#Y?`!b>s<9Ob(r?3m3g=zQfypMqT<+cPn9HhZJSyLRn(nfL&Pd}mr@zS8s{ z**x!Y1BPgrV}?~vZg6pUStu_@G(7d{Vllz3c#D?E1eSQUA7v>q&f~48kC6Fn+B^9n zDWA`uQ_VzUJ@(H!Nb>lR2sANx@|lrJ%D6OYMAflLD5EP0WWBg@@bx$~?gbrOjN}p)dsTDSIA6L3?R5vvP|MnhDz&FOa@$ zXIbZ60Vegh2|j!3?D3H-kLMM+D+?ZXupb~w5g?CK&8OMjt7s^7wJrA7#Kp6`YCxABU z&l!X6gZrqu0O(Ymp>t=woAw`b{1d}31R{nILu)&;KQR4bvm6*=Xaa2ku>PG%zxj6e zJo=O0s6crIIRHpd@Q;yx6Soo<>K&ZYZ*=^2B%uUiMTN>8+746#v7mzPIRRASM!k0$ ze}lQ>*U#w#|B3OQfq%6Aw;1n6{RQ*f&UWNqQ# z48B(+QLvM-qlG=h&Jh5*=haV5xdVi1nuVhiM8w?C5x~X@lr{WS2C=dN)h$dQ=1$tE ztXy|<`giZ2EqCA059+_`@mt-mb}rUC`v3LK!Eq;j==;CGao6|$!*U1l6Y`Hb)?c{q ze!1D$e)i+wME%)|?XCt2_1ApvuAn$rSnh1l7J8CoVQU7Rwym(Olf|Fc%FafRJIlY* z9H^%Jsg!r1e;0$6@^dK-jo-xZ+|JtMA2I+zA(j4j8F2Ib7b3W$0{Z?h7S_Y3{p>&PE^^p{rsu3^7={+j%q%|jRH zd#*q^^9N`DL5&E4?#SQga>Vp;1Jg{0U5{j zAXr#f7{4q4A^c-J-1cQ0m36#F3OXqW>hB%iN4|ewR`{vNM#0LZ_!i=IR(zG}bWrHz zFfak%;jl0!Mlb)a$YjNk!9`}ha>i|YU|VU9IG-SScM3v1>) zua3Evde?_ZYn+1`o6jhY?W3?xlE+Q!NN^!Zxwc{GK%uUhpo4$o!-3v`PmjS(zlKrN zj!k{A(mU2eo5=skbb}r%aQ{sG)CV4(eB`fi zpLuBw6KJf)=stjp);VWS(){RYNw-QRgt&v_1L*VSFX9D&e zW||n1K)w%gJBVr}@>(D2l6;`4|E}@Y{v4_LiDg(({Pvk;i?8%zA4xNj(Pr$b3qX?A z`SdLWvCl$(cgyd%)Y7j!bJKfGIfF5eLM8iKkYdVyXCi%`Z~QgOwWU!@j-5pILl&py zt9MKcob&K(pJ9zr_#g)nhIxBk4ea>X8~RX4?gqS zYOiZ8+()&TUnf^OYHkx176OW=rA7SO9`ae2q z&Pq_hc&fF{*7t^hEVtecjrADQjPZ462dKBu0VT6A^V$A8HIn1WfE|hztz&E)Zm^pAT=z2y8UDMGNCCpo;|-0e$IDm zfK(E}`$weFd-2r#A%Zs}q(-k(h=L>a*uq^SvLmNS8*3usBcl;CpLONo?XcTRzsC|3 zc0DeJ$q4?uw(x^M++?@Q9vi{f#86Jy!eYDOgiG2EcG5wHC^FVFyelC!hFprj@tQDQ zGX_77j1e>Z4ax&M%kjrwMc0NAs3TrC8rh6~N8Cx!jMLN@Y>Q(f?kU4g@&6HhVHw)? zjtw|ztoxBSI}8A9E@vZZiYVds}*AffxZkZBjXk$C!_W#X8TZszSbM z2mJ_+rMe8HGh!ur{1u38BSBu4_-J*JanR7`@jg#@ZRJ$I>H9h)seB{1fg*WOCtKx9 z|7A`-y$_j$)dzt$b6?^PZk~#^Qw7NGcgIPe=2CYdM~y3wua9CXL`+(VcaxZ-{z!JM zY62`8kXs)ishY0_Lup76rj7L8>-Z7R z?WxqTnmLbsfxfu#ij(=xo=e7R=XI%#cQ*tplR@umQlANYz~68peY0VTZ0654h8E84 z734hh&FHwC($_Y6yxk&<3h8w%O@BE2NlFgE_=#XXrXwh8vg3Sa^TE!u!san7IXs=b zB|B1+^6w+W zjvApm!Dnr+@TwL0oUfxeIpD%H-)7J;+Hssb zd2Wuxzns8Jl`9z`Fm7kavNN|{&2C#4o-4HXXxUkp!0tqHZs@5=b$&dAOM%(&J&?Cr zeumM)&7*Fmcavwxgm0HL#}d5w&E%`5MePDMJIvA7@Da^`sR({bVY?Ki`ENpo!;`NpS5{Z9DG ze)`h`DocAw4p)?Omm@MRgXk8XWX{@kJF$imPD)Nn0osZ3=~i=*OBj?#cKS-em~bXN z{%;2OYL8RWvgxL_G7~T-EjWUPy41Vy170zlfAYARK&|P5G1g*Ne!EjQ zsy0nJ-NH1JS6)4iJe|#Zv$xfU4RqE%t4u5>EUc%O*1!{ZO)yIN5by4u>0ggvpkD2t zcLDc4+}+LSzw@d7^BvW_L&XC1iJ%TO7w7NZ6erZ3h5AYVl|#k$pZqBh=fCr({&b;) zfzP3ClqgUPC;^lNNK+a)vCH$Bfp#C|H= zIWPIiAySGYE}>52q>`1Di8q}iiS(bKETzYI|_ITA31dG`( zOnKF$GB4xkoUJ}E(^1pt6BI`?pX(!#uj4nk?MLM*j)MA)*H+a_rDe5R#Dhi{*+~M( z<<|qg^&nkPz#U77vlAd>JY&ZklxQ=DS$)Ts9lUOi#Typbz+#K(y42NAJ)1kSkW#*f~*6DZ5unO~kCQIzCd!x9#V`#)OwVsVl*N%V!5=^<8uIL7Gf`Tozt z)^8$Y=9@KSvJq;$m}_UVtx4y>?;;-+5Uj984U8lET5jv^pt-seGd~OQo7`#3Fi_X} z7_0J{Z~7DO-eDwcMb2i}m%X?gK3}F7<9X#mzVCT_+ZguQ9+H8086GGrs#-+^E<`wj z?5x48m!JW^Wo@4h|DeZo!kq7;`FoKbPIm{pR*ZT@uY^BfeOa{?;rZF*#NB- zy83fEWmSqyCH)Tq>1^Lh@}P%6Bv-^g8gMBJ%{|l3@Fo5vi_S?^kU46h_O_Fz-xfPn z{μoTXP0o`4o>pX=w+)V0_dIQ(kahTWTtXl^*3akBO@L2Z(3Ji5 zgUF?urNfK3H?bbMB0x#3Ss`~)OZz>1bB5VLSH}AN^=NOV4!pu26-{4)TQb9oGCvR%lF(z(mdjhDxnm z53>SU?258>g7XdF*RJz3$hf8vn-X7{e%TjrnSlvWftd&{S9)dP^X3)i7J*=flg8_A z%}0tu<-A>fc>_IX<7^!@FEDpzf<-xBhae8FwF?&`AELE%JE>dBg|-CDgu1SRHxMM) zVjul@Z%{}l=r14B{$&jA)0Z|Jrp0rt56$n{PQK+G3h|Hz#bRJ?)&h7)nsgj6p3n)X zL8=SrG0j#6jH+tPuQmgyNy?+J%3p;}ybk5VtXiQtiisr1(OE$Az7#TNlo`hMqD>XM zD4kq3o)VlttuF5kSC@;Bm&QDj=AYJQ^}CKOGXE%q1|VZF#Oon@4Wz~P&Ln%NML)4^ zS!Lz0Igvgu|NSk*HoPQ(e4|4gkx+Gpe3l|g5vx+;8F+ZPs3MN*A!= zQ%Y1AEnVXJ+ z0jT)r7N!PN2WkMdps@l&2sB(^`+?|`;vR*aeFFUjr2ob4M^VVBu;9bOt)xnt&ahjO`r3Ko_7ZG=||0 z^ZqIZlXhigGQt>;rPsuA?X+kx)??WU<^4xmu}EHhzRlXS+nQV!R^iW zgMsbV(%Z%lR<639EkP|)RR^2i2b+4MF^ZU>c@}H$CI)^?G&l50jm5T@%Z0&t2K$+W z*-srtI|rb%Mv~)X7x{W&Y6ZhxeYA8sI*{u%pA+Kb0r2IsxY~M%r$va!u76vh6z9Hn z?q0OX$sbH<7fPFf&tpzWP6HduBVH6H&hivZlr#r76!C=N&xullnW|B$LS*IiV-jPo z-X`OU1zehW@sslbXawzr6%INB+T6**o^Ax;Q_1P^)0D9AG}BU3aM)O0R)_aAD4t|( zcAI=E%ENTx=6v}h96SGe$Pjat&Z2mu=S!0{4$~4Yu~{YM0#M9jj#%R7a@oy=nJQFZ zc`?O~0`orK(9A&PiWo|cRAON-S-Fgq0&ec8Fs6)0s zA(1gO%Whr+i8FGs{ljv2eeZ%x>6@zv%uRxm?*g8u;aeBk6?I>+*G~0x%Et7jJ)M&@ z5<3%3lC3%2jd>T?2Ozl?i597|>ZY56a)YUZVFRuSPqmTFOqp?w*LH z9}5a;xM_-Nmb|7kL&stlA_2fi;eaSbOhcT80g`ZX<@yRSGtK*K(-d_3!wuCY774dql=3G)VhVn*P>ApB}{5lkA zT!V?2A%pNamrRC->=yiJ)q$h9%N1S&m3%=AVm^7(g@pP+A=4yAr$kHPT@$y}B;;5L z*_-|u9j^WHS<(*tlh5b=AnD5e=BJ~DA%deAHT$eHz_dr@enCtaox+b@SV&3JT7tel zx*XNs%VGSAlcxW*3?QVxK0G5PfUm~4DpK9;`4)`?c)kfP{we78nUV|jMc z3zJW>iMa1APZiRC}&uU8`e$AO~MwOj)&)&z;iR{NVl&_W|MzbuxO&0S-MCkqTlCCQ%a#wg$C#=DA4Dma9Nsd&S51GjGlzW6^_02C+12Vof?OG^WBQE62;g|2f6gHV&%yfW##JC}}aQ$)rS_ef#rx|vE z_(Mc=(|e$vX5|7e;$WgsM|`}zSDRE%LFT~|hL0onzlFAwS6{Lj#6YxXXyszD`(V6&o-Z5t+rgy4$?DwFZm}H@|~_T#O?pE-rS6B zM$!(I+s$tSM0MP}ceYD1^RlYb-)>l45)k6buG?q%y9e8Z_|{)RA<0aB9$Nh7V**c3jHkKlP-R|$FkA0Qyh%JNuDph@)Nvk4W?P(fK!V2g6 z7aOm4zqe#81-wGEN$Cv!#7)P9@64ny$9m(N`?4b8v;2Yp@wEMM6{n(bT6ENhhQdb; zqaH{@Rm?kF0sQ4Cs!vKj^vf?x`Y%SETsEK^q1dE0sSM}p5VFd3^m@OeJupqt+{-xY z;>-dPJx}oFqRcJUG%L>Q*Z&9`HI|RP(a+7kU*#^=5HRZ$HumUAjAN;s`;Y}|P(*UK z$dh+M9`N|6UuDlub1u5ju0M9Mw5Bdu1aEbsgM+qHt$RMkBG6P0IGsjpT>@JejA!z< zrbN?igB~-ne%-Y+?3$b0a_Tf=nstAMB4j_eOw4#L#wabN&N!f1s>PRT2j=?B>s~ki#*`CK^1Kig+r-$Y+TX~<<2x6cyu_Nlahsud20W&V?rcB` zv+8c$bRGl#R0M(Sj@yYM{+U!aOMTYZDudKed< zKSeSoZbVGx8!J*UA^L(Ba^%X!^Ydy@k_s1Lm~3@X@qw~hEtO5mH8 z+GmLg)>EaN$0g=PhXzX|Q48?o-C~rd1aL2(J6H#DHY5-ozwymfJP3x{tPopL4>o(1Xp4VDMr^E)L67*J2ML>`u zGi~6Js%G5`-hu6{g zX)RHa;?5$ko}be5YU?@w7~pCe%H-omC!Y5YqCKLP>Jqb}oI$NvsG1{MTB=}MpnkJb zg1ALF`U>)XZQs7RS4mI-0x%3;TQ9SHO06ba`N{lb->1x zD^G9Svj^~7+9yYaV;N^rb=-w`uGlU5v^3 zIv!w?jXq@RyzM1M=a_rGkiLFRk5Qsiq_)Sv*e_=BvJPnvxru+m$8>4k4T1CqaJ5NA zBVkpOg(b`n8%(3!Rjcy!tx74H7+c-ITxKP#DKNRhZqm1=Y|6KH6-Tg}J^zhbwi+4Z z%R}ixH&e-g2rLK7u;4_#W=FfWn;(x)My!{d)I$B*(H$RBe7kJv+MgQr-_x?E+9rJe zd9l6k^OlOZUhVs+T&j!~i#(EQ3c1U-Y75L6NgVo&JaG%WJLsG=G2bABepgNDVIGOa z{!(n!X!5z4k_<6X=_w7g3_=bUD(ewOuHclRi_7RWYBMQMW|Udbu7cxW=M?(8OqXxj zmkS{~MU>7|lc|bzk1f4kR@O~PvVBS6C*RB0Q4S`=5ObqyKj3oMvRX}I zt|Buaz!XI0$8uBgvqPG}PQHO(B{u(xTY%st^-iPRI{_Ist*4q>#O+}m7ddX5B>aKl zBVKgI3-emIc~#>lqC@C2lw?;P2CSop$n%`Onj%|F=1mB-a`@eI2Il$oiQnXbsa5rA z8timuS2-f1IVDI2P76;kYpjk&r@Wi;zwqTYe_wkai)O1S^lTbax8D0=DzbYE3$w}r z{e0PI0ZqHp(~~x^NxlsZ{S-)Gx*AfxeUqJvM1nW#D!wUJ7ECdoXAC4nRDAbT5oL^= zn7`uTY1Wc%gMNwz8Bnm%d4~Pco55>q^%kU9=!&O<^`)4E_3bdyIYzB zc7BTEcJHH-*1|m-qtCXq{cA{&^SD=ievq#&JzDIod!q4J>Gsk~Av+)sAunpq8o2TkF2t*FoQR>ecvoMf`~LTXft)bci?OR&Nw zz3K|5`an7$4Ts}L1+vf}c3G6cjSoO;@BJcaay$7^M;MOvy^)jK+%U@v9Hg02+}EkkqW zlUf{0nWbrjdDvec(`VT?1ErYi8!(`=r(NXFv;HE zI6O#*u<4wdu}uMzRfvza#l6&Sw3{b$o=o(L$NK!z_Dg;JbJafVjrNVi;*M$O^zzIW zBAt1Q(d0c~kU`O5+ozo?ko=7DWOrW~;HIlaVLns2L*krTMPP^qWSy=S*W%g61v+DW z5>2?L*9s?5?%*DR^obg%JgBJlj2~_@m*0hJM!$DrW2{;;Ei0w#!+BN63^_i*$?z;a ztsCNa8_pN+V5&+YTlP^JzLj+uO{Q?{KG3Z_g~Aq=WdXm}?trc{A1sV;S$mKamvU*~ zlgJw%gMDsv28Y`h1Y4f@9-lW3V|L4ZCXhXG-EOnYH*e9hyU*HRLW!4qvsUgrm8A`IK__OZ4iygSpkFTRY8-eLUAg|DOF(&slt9 zo3ioT>U|}Laqs}46B4;lGHn))Ua0DxR zyihUrurlT`p%D^9%{9OKywem3oYMO18D&FT_9EhG#YXW0EnHPwUq@l)s~ru8N>u)V+5EwLerRB z?HsKDAZAV)C`v~YQ(h%#qVi9K&?^BNa|pzq7YKB9b!B#CW43cN1A=&Xcz`UdKJAr=~cg4sGT-_yhlO*{fxXaGUXEWkeo`s4V|!p;_OH?X0R zoijv$#@X4zgcodNXaweBV}p`u0{sG+vNEwiUrbzF?5td1LsL#}F4I4Tv;U3Ee;E(j z03DDG$_HqP7n+G};%p3d6rhn%kOfGX+c`mgrpW@J?BJmR{+j;(6;)8#{Ch4y1^SB( z{LntUa?mVPQ+H8l9=RYZ3y6z}1u7a26;@VWb`D-P7A6*MUKSR9;NN;eg%1T4f&K*w z^e<8V3(((s{swC2r~*|9K{H1?Q&WHoG-7%u2@sSS%&b30ya)T$iywF=qu;pmf6XsL zdpWuPnPWCKcPf3?k5|;r*x3e}A}1yK|3+c{uVrjv{3pTo&W_gi)?s1{1Y3jeXmf&! z83d)+#F*FA&e6sYs@sP4_Rv&5LkN`7K&VdA2=YT4Ar{aOyWqX;30p&c7OXAqY>C&} z(AG?V#*N7YY-;Fi4WSWKwY~3c@>3et7J@$u_i(_!aQz!u-B&?v8!9&7pDTwTD&YU< z^?!%xujLY&MIb=qdbb$nM+)2=!KTp6SXNHdy9Mj#3jobfVB-Lo0)EwT zf}oN3`zCfV0hUQZIYaPp7X7b>DZ9%%TNIu=fLwttMv#m)B5xv_Hn zgJw=PmVeZ7@UZ`b7B()He~iJw#raQK*f^j$?|-9@jr$*U+@OEZ2V!C8{yWVeR(7_3 z(89&_4<0}j?05M>91WqlvyRX|wgLc^Ej++b-2gxz9kPQy@B>YXQ33#^Y)$O|cNz_S z5J*ZCpv}+C0up0o%=H|^$;v7!#?8*fEiS^v!NMi@zxU9+d!EJ#V(19D TS4a>yCp#x9HMN+$IO_icwA6bT literal 0 HcmV?d00001 diff --git a/contrib/zlib/zlib.h b/contrib/zlib/zlib.h index 3e0c7672a..577d81e3b 100644 --- a/contrib/zlib/zlib.h +++ b/contrib/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.8, April 28th, 2013 + version 1.2.11.1, January xxth, 2017 - Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -37,12 +37,12 @@ extern "C" { #endif -#define ZLIB_VERSION "1.2.8" -#define ZLIB_VERNUM 0x1280 +#define ZLIB_VERSION "1.2.11.1-motley" +#define ZLIB_VERNUM 0x12b1 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 2 -#define ZLIB_VER_REVISION 8 -#define ZLIB_VER_SUBREVISION 0 +#define ZLIB_VER_REVISION 11 +#define ZLIB_VER_SUBREVISION 1 /* The 'zlib' compression library provides in-memory compression and @@ -65,7 +65,8 @@ extern "C" { with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. - This library can optionally read and write gzip streams in memory as well. + This library can optionally read and write gzip and raw deflate streams in + memory as well. The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- @@ -74,7 +75,7 @@ extern "C" { The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash - even in case of corrupted input. + even in the case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); @@ -87,7 +88,7 @@ typedef struct z_stream_s { uInt avail_in; /* number of bytes available at next_in */ uLong total_in; /* total number of input bytes read so far */ - Bytef *next_out; /* next output byte should be put there */ + Bytef *next_out; /* next output byte will go here */ uInt avail_out; /* remaining free space at next_out */ uLong total_out; /* total number of bytes output so far */ @@ -98,8 +99,9 @@ typedef struct z_stream_s { free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ - int data_type; /* best guess about the data type: binary or text */ - uLong adler; /* adler32 value of the uncompressed data */ + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ } z_stream; @@ -142,7 +144,9 @@ typedef gz_header FAR *gz_headerp; zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be - thread safe. + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). On 16-bit systems, the functions zalloc and zfree must be able to allocate exactly 65536 bytes, but will not be required to allocate more than this if @@ -155,7 +159,7 @@ typedef gz_header FAR *gz_headerp; The fields total_in and total_out can be used for statistics or progress reports. After compression, total_in holds the total size of the - uncompressed data and may be saved for use in the decompressor (particularly + uncompressed data and may be saved for use by the decompressor (particularly if the decompressor wants to decompress everything in a single step). */ @@ -200,7 +204,7 @@ typedef gz_header FAR *gz_headerp; #define Z_TEXT 1 #define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ #define Z_UNKNOWN 2 -/* Possible values of the data_type field (though see inflate()) */ +/* Possible values of the data_type field for deflate() */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ @@ -258,11 +262,11 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); enough room in the output buffer), next_in and avail_in are updated and processing will resume at this point for the next call of deflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. This action is forced if the parameter flush is non zero. Forcing flush frequently degrades the compression ratio, so this parameter - should be set only when necessary (in interactive applications). Some - output may be provided even if flush is not set. + should be set only when necessary. Some output may be provided even if + flush is zero. Before the call of deflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more @@ -271,7 +275,9 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK and with zero avail_out, it must be called again after making room in the output - buffer because there might be more output pending. + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to decide how much data to accumulate before producing output, in order to @@ -292,8 +298,8 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. This completes the current deflate block and follows it with an empty fixed codes block that is 10 bits long. This assures that enough bytes are output - in order for the decompressor to finish the block before the empty fixed code - block. + in order for the decompressor to finish the block before the empty fixed + codes block. If flush is set to Z_BLOCK, a deflate block is completed and emitted, as for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to @@ -319,34 +325,38 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was - enough output space; if deflate returns with Z_OK, this function must be - called again with Z_FINISH and more output space (updated avail_out) but no - more input data, until it returns with Z_STREAM_END or an error. After - deflate has returned Z_STREAM_END, the only possible operations on the stream - are deflateReset or deflateEnd. + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. - Z_FINISH can be used immediately after deflateInit if all the compression - is to be done in a single step. In this case, avail_out must be at least the - value returned by deflateBound (see below). Then deflate is guaranteed to - return Z_STREAM_END. If not enough output space is provided, deflate will - not return Z_STREAM_END, and it must be called again as described above. + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. - deflate() sets strm->adler to the adler32 checksum of all input read - so far (that is, total_in bytes). + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) deflate() may update strm->data_type if it can make a good guess about - the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered - binary. This field is only for information purposes and does not affect the - compression algorithm in any manner. + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. deflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if all input has been consumed and all output has been produced (only when flush is set to Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example - if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible - (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not - fatal, and deflate() can be called again with more input and more output - space to continue compressing. + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. */ @@ -369,23 +379,21 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by - the caller. If next_in is not Z_NULL and avail_in is large enough (the - exact value depends on the compression method), inflateInit determines the - compression method from the zlib header and allocates all data structures - accordingly; otherwise the allocation will be deferred to the first call of - inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to - use default allocation functions. + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the version assumed by the caller, or Z_STREAM_ERROR if the parameters are invalid, such as a null pointer to the structure. msg is set to null if - there is no error message. inflateInit does not perform any decompression - apart from possibly reading the zlib header if present: actual decompression - will be done by inflate(). (So next_in and avail_in may be modified, but - next_out and avail_out are unused and unchanged.) The current implementation - of inflateInit() does not process any header information -- that is deferred - until inflate() is called. + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. */ @@ -401,17 +409,20 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); - Decompress more input starting at next_in and update next_in and avail_in accordingly. If not all input can be processed (because there is not - enough room in the output buffer), next_in is updated and processing will - resume at this point for the next call of inflate(). + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. inflate() provides as much output as possible, until there is no more input data or no more space in the output buffer (see below about the flush parameter). Before the call of inflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more - output, and updating the next_* and avail_* values accordingly. The + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The application can consume the uncompressed output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of inflate(). If inflate returns Z_OK and with zero avail_out, it must be @@ -428,7 +439,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); gets to the end of that block, or when it runs out of data. The Z_BLOCK option assists in appending to or combining deflate streams. - Also to assist in this, on return inflate() will set strm->data_type to the + To assist in this, on return inflate() always sets strm->data_type to the number of unused bits in the last byte taken from strm->next_in, plus 64 if inflate() is currently decoding the last block in the deflate stream, plus 128 if inflate() returned immediately after decoding an end-of-block code or @@ -454,7 +465,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); this case all pending input is processed and all pending output is flushed; avail_out must be large enough to hold all of the uncompressed data for the operation to complete. (The size of the uncompressed data may have been - saved by the compressor for this purpose.) The use of Z_FINISH is not + saved by the compressor for this purpose.) The use of Z_FINISH is not required to perform an inflation in one step. However it may be used to inform inflate that a faster approach can be used for the single inflate() call. Z_FINISH also informs inflate to not maintain a sliding window if the @@ -476,32 +487,33 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); chosen by the compressor and returns Z_NEED_DICT; otherwise it sets strm->adler to the Adler-32 checksum of all output produced so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described - below. At the end of the stream, inflate() checks that its computed adler32 + below. At the end of the stream, inflate() checks that its computed Adler-32 checksum is equal to that saved by the compressor and returns Z_STREAM_END only if the checksum is correct. inflate() can decompress and check either zlib-wrapped or gzip-wrapped deflate data. The header type is detected automatically, if requested when initializing with inflateInit2(). Any information contained in the gzip - header is not retained, so applications that need that information should - instead use raw inflate, see inflateInit2() below, or inflateBack() and - perform their own processing of the gzip header and trailer. When processing + header is not retained unless inflateGetHeader() is used. When processing gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output - producted so far. The CRC-32 is checked against the gzip trailer. + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. inflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if the end of the compressed data has been reached and all uncompressed output has been produced, Z_NEED_DICT if a preset dictionary is needed at this point, Z_DATA_ERROR if the input data was corrupted (input stream not conforming to the zlib format or incorrect check - value), Z_STREAM_ERROR if the stream structure was inconsistent (for example - next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory, - Z_BUF_ERROR if no progress is possible or if there was not enough room in the - output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and inflate() can be called again with more input and more output space to continue decompressing. If Z_DATA_ERROR is returned, the application may then call inflateSync() to look for a good compression block if a partial - recovery of the data is desired. + recovery of the data is to be attempted. */ @@ -511,9 +523,8 @@ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); This function discards any unprocessed input and does not flush any pending output. - inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state - was inconsistent. In the error case, msg may be set but then points to a - static string (which must not be deallocated). + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. */ @@ -544,16 +555,29 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, compression at the expense of memory usage. The default value is 15 if deflateInit is used instead. + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits determines the window size. deflate() will then generate raw deflate data - with no zlib header or trailer, and will not compute an adler32 check value. + with no zlib header or trailer, and will not compute a check value. windowBits can also be greater than 15 for optional gzip encoding. Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib wrapper. The gzip header will have no file name, no extra data, no comment, no modification time (set to zero), no - header crc, and the operating system will be set to 255 (unknown). If a - gzip stream is being written, strm->adler is a crc32 instead of an adler32. + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. The memLevel parameter specifies how much memory should be allocated for the internal compression state. memLevel=1 uses minimum memory but is @@ -614,12 +638,12 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, addition, the current implementation of deflate will use at most the window size minus 262 bytes of the provided dictionary. - Upon return of this function, strm->adler is set to the adler32 value + Upon return of this function, strm->adler is set to the Adler-32 value of the dictionary; the decompressor may later use this value to determine - which dictionary has been used by the compressor. (The adler32 value + which dictionary has been used by the compressor. (The Adler-32 value applies to the whole dictionary even if only a subset of the dictionary is actually used by the compressor.) If a raw deflate was requested, then the - adler32 value is not computed and strm->adler is not set. + Adler-32 value is not computed and strm->adler is not set. deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is @@ -628,6 +652,28 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, not perform any compression: this will be done by deflate(). */ +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, z_streamp source)); /* @@ -648,10 +694,10 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* - This function is equivalent to deflateEnd followed by deflateInit, - but does not free and reallocate all the internal compression state. The - stream will keep the same compression level and any other attributes that - may have been set by deflateInit2. + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). @@ -662,20 +708,37 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, int strategy)); /* Dynamically update the compression level and compression strategy. The - interpretation of level and strategy is as in deflateInit2. This can be + interpretation of level and strategy is as in deflateInit2(). This can be used to switch between compression and straight copy of the input data, or to switch to a different kind of input data requiring a different strategy. - If the compression level is changed, the input available so far is - compressed with the old level (and may be flushed); the new level will take - effect only at the next call of deflate(). + If the compression approach (which is a function of the level) or the + strategy is changed, and if there have been any deflate() calls since the + state was initialized or reset, then the input available so far is + compressed with the old level and strategy using deflate(strm, Z_BLOCK). + There are three approaches for the compression levels 0, 1..3, and 4..9 + respectively. The new level and strategy will take effect at the next call + of deflate(). - Before the call of deflateParams, the stream state must be set as for - a call of deflate(), since the currently available input may have to be - compressed and flushed. In particular, strm->avail_out must be non-zero. + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. - deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source - stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if - strm->avail_out was zero. + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. */ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, @@ -793,7 +856,7 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, is for use with other formats that use the deflate compressed data format such as zip. Those formats provide their own check values. If a custom format is developed using the raw deflate format for compressed data, it is - recommended that a check value such as an adler32 or a crc32 be applied to + recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib, gzip, and zip formats. For most applications, the zlib format should be used as is. Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. @@ -802,7 +865,10 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, 32 to windowBits to enable zlib and gzip decoding with automatic header detection, or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - crc32 instead of an adler32. + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will not automatically decode concatenated gzip streams. + inflate() will return Z_STREAM_END at the end of the gzip stream. The state + would need to be reset to continue decoding a subsequent gzip stream. inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the @@ -823,7 +889,7 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, if that call returned Z_NEED_DICT. The dictionary chosen by the compressor - can be determined from the adler32 value returned by that call of inflate. + can be determined from the Adler-32 value returned by that call of inflate. The compressor and decompressor must use exactly the same dictionary (see deflateSetDictionary). For raw inflate, this function can be called at any time to set the dictionary. If the provided dictionary is smaller than the @@ -834,7 +900,7 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the - expected one (incorrect adler32 value). inflateSetDictionary does not + expected one (incorrect Adler-32 value). inflateSetDictionary does not perform any decompression: this will be done by subsequent calls of inflate(). */ @@ -892,7 +958,7 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, - but does not free and reallocate all the internal decompression state. The + but does not free and reallocate the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source @@ -904,7 +970,9 @@ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted - the same as it is for inflateInit2. + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL), or if @@ -956,7 +1024,7 @@ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); location in the input stream can be determined from avail_in and data_type as noted in the description for the Z_BLOCK flush parameter for inflate. - inflateMark returns the value noted above or -1 << 16 if the provided + inflateMark returns the value noted above, or -65536 if the provided source stream state was inconsistent. */ @@ -1048,9 +1116,9 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, This routine would normally be used in a utility that reads zip or gzip files and writes out uncompressed files. The utility would decode the header and process the trailer on its own, hence this routine expects only - the raw deflate stream to decompress. This is different from the normal - behavior of inflate(), which expects either a zlib or gzip header and - trailer around the deflate stream. + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. inflateBack() uses two subroutines supplied by the caller that are then called by inflateBack() for input and output. inflateBack() calls those @@ -1059,12 +1127,12 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, parameters and return types are defined above in the in_func and out_func typedefs. inflateBack() will call in(in_desc, &buf) which should return the number of bytes of provided input, and a pointer to that input in buf. If - there is no input available, in() must return zero--buf is ignored in that - case--and inflateBack() will return a buffer error. inflateBack() will call - out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() - should return zero on success, or non-zero on failure. If out() returns - non-zero, inflateBack() will return with an error. Neither in() nor out() - are permitted to change the contents of the window provided to + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to inflateBackInit(), which is also the buffer that out() uses to write from. The length written by out() will be at most the window size. Any non-zero amount of input may be provided by in(). @@ -1092,7 +1160,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, using strm->next_in which will be Z_NULL only if in() returned an error. If strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning non-zero. (in() will always be called before out(), so strm->next_in is - assured to be defined if out() returns non-zero.) Note that inflateBack() + assured to be defined if out() returns non-zero.) Note that inflateBack() cannot return Z_OK. */ @@ -1114,7 +1182,7 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 7.6: size of z_off_t Compiler, assembler, and debug options: - 8: DEBUG + 8: ZLIB_DEBUG 9: ASMV or ASMINF -- use ASM code 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention 11: 0 (reserved) @@ -1164,7 +1232,8 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. compress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1180,7 +1249,7 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, @@ -1203,7 +1272,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some mechanism outside the scope of this compression library.) Upon exit, destLen - is the actual size of the uncompressed buffer. + is the actual size of the uncompressed data. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1212,6 +1281,14 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, buffer with the uncompressed data up to that point. */ +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + /* gzip file access functions */ /* @@ -1290,10 +1367,9 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); default buffer size is 8192 bytes. This function must be called after gzopen() or gzdopen(), and before any other calls that read or write the file. The buffer memory allocation is always deferred to the first read or - write. Two buffers are allocated, either both of the specified size when - writing, or one of the specified size and the other twice that size when - reading. A larger buffer size of, for example, 64K or 128K bytes will - noticeably increase the speed of decompression (reading). + write. Three times that size in buffer space is allocated. A larger buffer + size of, for example, 64K or 128K bytes will noticeably increase the speed + of decompression (reading). The new buffer size also affects the maximum length for gzprintf(). @@ -1304,10 +1380,12 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* Dynamically update the compression level or strategy. See the description - of deflateInit2 for the meaning of these parameters. + of deflateInit2 for the meaning of these parameters. Previously provided + data is flushed before the parameter change. - gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not - opened for writing. + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. */ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); @@ -1335,7 +1413,35 @@ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); case. gzread returns the number of uncompressed bytes actually read, less than - len for end of file, or -1 for error. + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +/* + Read up to nitems items of size size from file to buf, otherwise operating + as gzread() does. This duplicates the interface of stdio's fread(), with + size_t request and return types. If the library defines size_t, then + z_size_t is identical to size_t. If not, then z_size_t is an unsigned + integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. */ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, @@ -1346,19 +1452,33 @@ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, error. */ +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +/* + gzfwrite() writes nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); /* Converts, formats, and writes the arguments to the compressed file under control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written, or 0 in case of error. The number of - uncompressed bytes written is limited to 8191, or one less than the buffer - size given to gzbuffer(). The caller should assure that this limit is not - exceeded. If it is exceeded, then gzprintf() will return an error (0) with - nothing written. In this case, there may also be a buffer overflow with - unpredictable consequences, which is possible only if zlib was compiled with - the insecure functions sprintf() or vsprintf() because the secure snprintf() - or vsnprintf() functions were not available. This can be determined using - zlibCompileFlags(). + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf() + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). */ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); @@ -1418,7 +1538,7 @@ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); If the flush parameter is Z_FINISH, the remaining data is written and the gzip stream is completed in the output. If gzwrite() is called again, a new gzip stream will be started in the output. gzread() is able to read such - concatented gzip streams. + concatenated gzip streams. gzflush should be called only when strictly necessary because it will degrade compression if called too often. @@ -1572,7 +1692,7 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); return the updated checksum. If buf is Z_NULL, this function returns the required initial value for the checksum. - An Adler-32 checksum is almost as reliable as a CRC32 but can be computed + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed much faster. Usage example: @@ -1585,6 +1705,12 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); if (adler != original_adler) error(); */ +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as adler32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, z_off_t len2)); @@ -1614,6 +1740,12 @@ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); if (crc != original_crc) error(); */ +ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as crc32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); @@ -1644,19 +1776,35 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, unsigned char FAR *window, const char *version, int stream_size)); -#define deflateInit(strm, level) \ - deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) -#define inflateInit(strm) \ - inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) -#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ - deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ - (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) -#define inflateInit2(strm, windowBits) \ - inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ - (int)sizeof(z_stream)) -#define inflateBackInit(strm, windowBits, window) \ - inflateBackInit_((strm), (windowBits), (window), \ - ZLIB_VERSION, (int)sizeof(z_stream)) +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif #ifndef Z_SOLO @@ -1676,10 +1824,10 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #ifdef Z_PREFIX_SET # undef z_gzgetc # define z_gzgetc(g) \ - ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) #else # define gzgetc(g) \ - ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) #endif /* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or @@ -1737,16 +1885,13 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #endif /* !Z_SOLO */ -/* hack for buggy compilers */ -#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) - struct internal_state {int dummy;}; -#endif - /* undocumented functions */ ZEXTERN const char * ZEXPORT zError OF((int)); ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); #if defined(_WIN32) && !defined(Z_SOLO) diff --git a/contrib/zlib/zlib.map b/contrib/zlib/zlib.map new file mode 100644 index 000000000..82ce98cf7 --- /dev/null +++ b/contrib/zlib/zlib.map @@ -0,0 +1,94 @@ +ZLIB_1.2.0 { + global: + compressBound; + deflateBound; + inflateBack; + inflateBackEnd; + inflateBackInit_; + inflateCopy; + local: + deflate_copyright; + inflate_copyright; + inflate_fast; + inflate_table; + zcalloc; + zcfree; + z_errmsg; + gz_error; + gz_intmax; + _*; +}; + +ZLIB_1.2.0.2 { + gzclearerr; + gzungetc; + zlibCompileFlags; +} ZLIB_1.2.0; + +ZLIB_1.2.0.8 { + deflatePrime; +} ZLIB_1.2.0.2; + +ZLIB_1.2.2 { + adler32_combine; + crc32_combine; + deflateSetHeader; + inflateGetHeader; +} ZLIB_1.2.0.8; + +ZLIB_1.2.2.3 { + deflateTune; + gzdirect; +} ZLIB_1.2.2; + +ZLIB_1.2.2.4 { + inflatePrime; +} ZLIB_1.2.2.3; + +ZLIB_1.2.3.3 { + adler32_combine64; + crc32_combine64; + gzopen64; + gzseek64; + gztell64; + inflateUndermine; +} ZLIB_1.2.2.4; + +ZLIB_1.2.3.4 { + inflateReset2; + inflateMark; +} ZLIB_1.2.3.3; + +ZLIB_1.2.3.5 { + gzbuffer; + gzoffset; + gzoffset64; + gzclose_r; + gzclose_w; +} ZLIB_1.2.3.4; + +ZLIB_1.2.5.1 { + deflatePending; +} ZLIB_1.2.3.5; + +ZLIB_1.2.5.2 { + deflateResetKeep; + gzgetc_; + inflateResetKeep; +} ZLIB_1.2.5.1; + +ZLIB_1.2.7.1 { + inflateGetDictionary; + gzvprintf; +} ZLIB_1.2.5.2; + +ZLIB_1.2.9 { + inflateCodesUsed; + inflateValidate; + uncompress2; + gzfread; + gzfwrite; + deflateGetDictionary; + adler32_z; + crc32_z; +} ZLIB_1.2.7.1; diff --git a/contrib/zlib/zlib.pc.in b/contrib/zlib/zlib.pc.in new file mode 100644 index 000000000..7e5acf9c7 --- /dev/null +++ b/contrib/zlib/zlib.pc.in @@ -0,0 +1,13 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +sharedlibdir=@sharedlibdir@ +includedir=@includedir@ + +Name: zlib +Description: zlib compression library +Version: @VERSION@ + +Requires: +Libs: -L${libdir} -L${sharedlibdir} -lz +Cflags: -I${includedir} diff --git a/contrib/zlib/zlib2ansi b/contrib/zlib/zlib2ansi new file mode 100644 index 000000000..15e3e165f --- /dev/null +++ b/contrib/zlib/zlib2ansi @@ -0,0 +1,152 @@ +#!/usr/bin/perl + +# Transform K&R C function definitions into ANSI equivalent. +# +# Author: Paul Marquess +# Version: 1.0 +# Date: 3 October 2006 + +# TODO +# +# Asumes no function pointer parameters. unless they are typedefed. +# Assumes no literal strings that look like function definitions +# Assumes functions start at the beginning of a line + +use strict; +use warnings; + +local $/; +$_ = <>; + +my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments + +my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; +my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; +my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; + + +while (s/^ + ( # Start $1 + ( # Start $2 + .*? # Minimal eat content + ( ^ \w [\w\s\*]+ ) # $3 -- function name + \s* # optional whitespace + ) # $2 - Matched up to before parameter list + + \( \s* # Literal "(" + optional whitespace + ( [^\)]+ ) # $4 - one or more anythings except ")" + \s* \) # optional whitespace surrounding a Literal ")" + + ( (?: $dList )+ ) # $5 + + $sp ^ { # literal "{" at start of line + ) # Remember to $1 + //xsom + ) +{ + my $all = $1 ; + my $prefix = $2; + my $param_list = $4 ; + my $params = $5; + + StripComments($params); + StripComments($param_list); + $param_list =~ s/^\s+//; + $param_list =~ s/\s+$//; + + my $i = 0 ; + my %pList = map { $_ => $i++ } + split /\s*,\s*/, $param_list; + my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; + + my @params = split /\s*;\s*/, $params; + my @outParams = (); + foreach my $p (@params) + { + if ($p =~ /,/) + { + my @bits = split /\s*,\s*/, $p; + my $first = shift @bits; + $first =~ s/^\s*//; + push @outParams, $first; + $first =~ /^(\w+\s*)/; + my $type = $1 ; + push @outParams, map { $type . $_ } @bits; + } + else + { + $p =~ s/^\s+//; + push @outParams, $p; + } + } + + + my %tmp = map { /$pMatch/; $_ => $pList{$1} } + @outParams ; + + @outParams = map { " $_" } + sort { $tmp{$a} <=> $tmp{$b} } + @outParams ; + + print $prefix ; + print "(\n" . join(",\n", @outParams) . ")\n"; + print "{" ; + +} + +# Output any trailing code. +print ; +exit 0; + + +sub StripComments +{ + + no warnings; + + # Strip C & C++ coments + # From the perlfaq + $_[0] =~ + + s{ + /\* ## Start of /* ... */ comment + [^*]*\*+ ## Non-* followed by 1-or-more *'s + ( + [^/*][^*]*\*+ + )* ## 0-or-more things which don't start with / + ## but do end with '*' + / ## End of /* ... */ comment + + | ## OR C++ Comment + // ## Start of C++ comment // + [^\n]* ## followed by 0-or-more non end of line characters + + | ## OR various things which aren't comments: + + ( + " ## Start of " ... " string + ( + \\. ## Escaped char + | ## OR + [^"\\] ## Non "\ + )* + " ## End of " ... " string + + | ## OR + + ' ## Start of ' ... ' string + ( + \\. ## Escaped char + | ## OR + [^'\\] ## Non '\ + )* + ' ## End of ' ... ' string + + | ## OR + + . ## Anything other char + [^/"'\\]* ## Chars which doesn't start a comment, string or escape + ) + }{$2}gxs; + +} diff --git a/contrib/zlib/zutil.c b/contrib/zlib/zutil.c index 23d2ebef0..dcab28a0d 100644 --- a/contrib/zlib/zutil.c +++ b/contrib/zlib/zutil.c @@ -1,5 +1,5 @@ /* zutil.c -- target dependent utility functions for the compression library - * Copyright (C) 1995-2005, 2010, 2011, 2012 Jean-loup Gailly. + * Copyright (C) 1995-2017 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -10,21 +10,18 @@ # include "gzguts.h" #endif -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - z_const char * const z_errmsg[10] = { -"need dictionary", /* Z_NEED_DICT 2 */ -"stream end", /* Z_STREAM_END 1 */ -"", /* Z_OK 0 */ -"file error", /* Z_ERRNO (-1) */ -"stream error", /* Z_STREAM_ERROR (-2) */ -"data error", /* Z_DATA_ERROR (-3) */ -"insufficient memory", /* Z_MEM_ERROR (-4) */ -"buffer error", /* Z_BUF_ERROR (-5) */ -"incompatible version",/* Z_VERSION_ERROR (-6) */ -""}; + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; const char * ZEXPORT zlibVersion() @@ -61,7 +58,7 @@ uLong ZEXPORT zlibCompileFlags() case 8: flags += 2 << 6; break; default: flags += 3 << 6; } -#ifdef DEBUG +#ifdef ZLIB_DEBUG flags += 1 << 8; #endif #if defined(ASMV) || defined(ASMINF) @@ -115,8 +112,8 @@ uLong ZEXPORT zlibCompileFlags() return flags; } -#ifdef DEBUG - +#ifdef ZLIB_DEBUG +#include # ifndef verbose # define verbose 0 # endif @@ -139,8 +136,8 @@ const char * ZEXPORT zError(err) return ERR_MSG(err); } -#if defined(_WIN32_WCE) - /* The Microsoft C Run-Time Library for Windows CE doesn't have +#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 + /* The older Microsoft C Run-Time Library for Windows CE doesn't have * errno. We define it as a global variable to simplify porting. * Its value is always 0 and should not be used. */ @@ -219,9 +216,11 @@ local ptr_table table[MAX_PTR]; voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) { - voidpf buf = opaque; /* just to make some compilers happy */ + voidpf buf; ulg bsize = (ulg)items*size; + (void)opaque; + /* If we allocate less than 65520 bytes, we assume that farmalloc * will return a usable pointer which doesn't have to be normalized. */ @@ -244,6 +243,9 @@ voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { int n; + + (void)opaque; + if (*(ush*)&ptr != 0) { /* object < 64K */ farfree(ptr); return; @@ -259,7 +261,6 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) next_ptr--; return; } - ptr = opaque; /* just to make some compilers happy */ Assert(0, "zcfree: ptr not found"); } @@ -278,13 +279,13 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; return _halloc((long)items, size); } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; _hfree(ptr); } @@ -306,7 +307,7 @@ voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) unsigned items; unsigned size; { - if (opaque) items += size - size; /* make compiler happy */ + (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } @@ -315,8 +316,8 @@ void ZLIB_INTERNAL zcfree (opaque, ptr) voidpf opaque; voidpf ptr; { + (void)opaque; free(ptr); - if (opaque) return; /* make compiler happy */ } #endif /* MY_ZCALLOC */ diff --git a/contrib/zlib/zutil.h b/contrib/zlib/zutil.h index 24ab06b1c..60a0bca79 100644 --- a/contrib/zlib/zutil.h +++ b/contrib/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -29,14 +29,12 @@ # include #endif -#ifdef Z_SOLO - typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ -#endif - #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ typedef unsigned char uch; typedef uch FAR uchf; @@ -98,28 +96,38 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif #ifdef AMIGA -# define OS_CODE 0x01 +# define OS_CODE 1 #endif #if defined(VAXC) || defined(VMS) -# define OS_CODE 0x02 +# define OS_CODE 2 # define F_OPEN(name, mode) \ fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") #endif +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + #if defined(ATARI) || defined(atarist) -# define OS_CODE 0x05 +# define OS_CODE 5 #endif #ifdef OS2 -# define OS_CODE 0x06 +# define OS_CODE 6 # if defined(M_I86) && !defined(Z_SOLO) # include # endif #endif #if defined(MACOS) || defined(TARGET_OS_MAC) -# define OS_CODE 0x07 +# define OS_CODE 7 # ifndef Z_SOLO # if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os # include /* for fdopen */ @@ -131,18 +139,24 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif -#ifdef TOPS20 -# define OS_CODE 0x0a +#ifdef __acorn +# define OS_CODE 13 #endif -#ifdef WIN32 -# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ -# define OS_CODE 0x0b -# endif +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 #endif -#ifdef __50SERIES /* Prime/PRIMOS */ -# define OS_CODE 0x0f +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 +#endif + +#ifdef __APPLE__ +# define OS_CODE 19 #endif #if defined(_BEOS_) || defined(RISCOS) @@ -152,10 +166,6 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX # if defined(_WIN32_WCE) # define fdopen(fd,mode) NULL /* No fdopen() */ -# ifndef _PTRDIFF_T_DEFINED - typedef int ptrdiff_t; -# define _PTRDIFF_T_DEFINED -# endif # else # define fdopen(fd,type) _fdopen(fd,type) # endif @@ -177,7 +187,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* common defaults */ #ifndef OS_CODE -# define OS_CODE 0x03 /* assume Unix */ +# define OS_CODE 3 /* assume Unix */ #endif #ifndef F_OPEN @@ -216,7 +226,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif /* Diagnostic functions */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include extern int ZLIB_INTERNAL z_verbose; extern void ZLIB_INTERNAL z_error OF((char *m)); From f6706f3532fcccfd58579d1dc70e049c06aee697 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Wed, 4 Oct 2017 20:50:16 +0300 Subject: [PATCH 220/305] Travis: Build with clang too --- .travis.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.travis.yml b/.travis.yml index d59689f78..2eab656cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,10 @@ branches: os: - linux +compiler: + - gcc + - clang + env: global: # COVERITY_SCAN_TOKEN @@ -34,6 +38,10 @@ env: matrix: exclude: - os: linux + compiler: gcc + env: + - os: linux + compiler: clang env: include: @@ -49,6 +57,12 @@ matrix: - os: linux compiler: gcc env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + - os: linux + compiler: clang + env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + - os: linux + compiler: clang + env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From 9eeece1b356320a4ecbd96ebecac4dad464f5437 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Wed, 4 Oct 2017 21:10:49 +0300 Subject: [PATCH 221/305] Travis: Treat warnings as errors, without typos this time --- .travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.sh b/.travis.sh index f19c3a000..1ab1ee2b1 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ERROR=ON -DASSIMP_ASAN=$ASAN + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_WERROR=ON -DASSIMP_ASAN=$ASAN } if [ $ANDROID ]; then From 452885672e314c7c0d2e938c454a00ada61877ac Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Wed, 4 Oct 2017 21:21:36 +0300 Subject: [PATCH 222/305] CMake: Remove OpenMP stuff, it's unused and breaks Travis clang build --- CMakeLists.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c0fe062e..0aae76837 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,19 +151,6 @@ IF(ASSIMP_DOUBLE_PRECISION) ADD_DEFINITIONS(-DASSIMP_DOUBLE_PRECISION) ENDIF(ASSIMP_DOUBLE_PRECISION) -# Check for OpenMP support -find_package(OpenMP) -if (OPENMP_FOUND) - SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - - IF(MSVC) - IF(MSVC_VERSION GREATER 1910) - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:twoPhase-") - ENDIF() - ENDIF() -endif() - CONFIGURE_FILE( ${CMAKE_CURRENT_LIST_DIR}/revision.h.in ${CMAKE_CURRENT_BINARY_DIR}/revision.h From 1ef3b0f3f3eb1d80bc787114f639eba750ddad2d Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 10:45:25 +0300 Subject: [PATCH 223/305] Fix warning about non-constant array size --- test/unit/TestModelFactory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/TestModelFactory.h b/test/unit/TestModelFactory.h index f848f5536..ca070890d 100644 --- a/test/unit/TestModelFactory.h +++ b/test/unit/TestModelFactory.h @@ -89,7 +89,7 @@ public: scene->mRootNode = new aiNode; scene->mRootNode->mNumMeshes = 1; - scene->mRootNode->mMeshes = new unsigned int[scene->mRootNode->mNumMeshes]{ 0 }; + scene->mRootNode->mMeshes = new unsigned int[1]{ 0 }; return scene; } From e7c112916b585a480f162cd07a6361ff392b292e Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Thu, 5 Oct 2017 20:11:18 +1100 Subject: [PATCH 224/305] Removed unnecessary files from zlib contribution --- contrib/zlib/.gitignore | 26 - contrib/zlib/ChangeLog | 1518 ------------------------------------ contrib/zlib/FAQ | 368 --------- contrib/zlib/INDEX | 68 -- contrib/zlib/Makefile.in | 410 ---------- contrib/zlib/configure | 921 ---------------------- contrib/zlib/make_vms.com | 867 -------------------- contrib/zlib/treebuild.xml | 116 --- contrib/zlib/zlib.3 | 149 ---- contrib/zlib/zlib.3.pdf | Bin 19320 -> 0 bytes contrib/zlib/zlib.map | 94 --- contrib/zlib/zlib.pc.in | 13 - contrib/zlib/zlib2ansi | 152 ---- 13 files changed, 4702 deletions(-) delete mode 100644 contrib/zlib/.gitignore delete mode 100644 contrib/zlib/ChangeLog delete mode 100644 contrib/zlib/FAQ delete mode 100644 contrib/zlib/INDEX delete mode 100644 contrib/zlib/Makefile.in delete mode 100644 contrib/zlib/configure delete mode 100644 contrib/zlib/make_vms.com delete mode 100644 contrib/zlib/treebuild.xml delete mode 100644 contrib/zlib/zlib.3 delete mode 100644 contrib/zlib/zlib.3.pdf delete mode 100644 contrib/zlib/zlib.map delete mode 100644 contrib/zlib/zlib.pc.in delete mode 100644 contrib/zlib/zlib2ansi diff --git a/contrib/zlib/.gitignore b/contrib/zlib/.gitignore deleted file mode 100644 index b1c7422fe..000000000 --- a/contrib/zlib/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -*.diff -*.patch -*.orig -*.rej - -*~ -*.a -*.lo -*.o -*.dylib - -*.gcda -*.gcno -*.gcov - -/example -/example64 -/examplesh -/libz.so* -/minigzip -/minigzip64 -/minigzipsh -/zlib.pc -/configure.log - -.DS_Store diff --git a/contrib/zlib/ChangeLog b/contrib/zlib/ChangeLog deleted file mode 100644 index 9c6d95b8d..000000000 --- a/contrib/zlib/ChangeLog +++ /dev/null @@ -1,1518 +0,0 @@ - - ChangeLog file for zlib - -Changes in 1.2.11.1 (xx Jan 2017) -- - -Changes in 1.2.11 (15 Jan 2017) -- Fix deflate stored bug when pulling last block from window -- Permit immediate deflateParams changes before any deflate input - -Changes in 1.2.10 (2 Jan 2017) -- Avoid warnings on snprintf() return value -- Fix bug in deflate_stored() for zero-length input -- Fix bug in gzwrite.c that produced corrupt gzip files -- Remove files to be installed before copying them in Makefile.in -- Add warnings when compiling with assembler code - -Changes in 1.2.9 (31 Dec 2016) -- Fix contrib/minizip to permit unzipping with desktop API [Zouzou] -- Improve contrib/blast to return unused bytes -- Assure that gzoffset() is correct when appending -- Improve compress() and uncompress() to support large lengths -- Fix bug in test/example.c where error code not saved -- Remedy Coverity warning [Randers-Pehrson] -- Improve speed of gzprintf() in transparent mode -- Fix inflateInit2() bug when windowBits is 16 or 32 -- Change DEBUG macro to ZLIB_DEBUG -- Avoid uninitialized access by gzclose_w() -- Allow building zlib outside of the source directory -- Fix bug that accepted invalid zlib header when windowBits is zero -- Fix gzseek() problem on MinGW due to buggy _lseeki64 there -- Loop on write() calls in gzwrite.c in case of non-blocking I/O -- Add --warn (-w) option to ./configure for more compiler warnings -- Reject a window size of 256 bytes if not using the zlib wrapper -- Fix bug when level 0 used with Z_HUFFMAN or Z_RLE -- Add --debug (-d) option to ./configure to define ZLIB_DEBUG -- Fix bugs in creating a very large gzip header -- Add uncompress2() function, which returns the input size used -- Assure that deflateParams() will not switch functions mid-block -- Dramatically speed up deflation for level 0 (storing) -- Add gzfread(), duplicating the interface of fread() -- Add gzfwrite(), duplicating the interface of fwrite() -- Add deflateGetDictionary() function -- Use snprintf() for later versions of Microsoft C -- Fix *Init macros to use z_ prefix when requested -- Replace as400 with os400 for OS/400 support [Monnerat] -- Add crc32_z() and adler32_z() functions with size_t lengths -- Update Visual Studio project files [AraHaan] - -Changes in 1.2.8 (28 Apr 2013) -- Update contrib/minizip/iowin32.c for Windows RT [Vollant] -- Do not force Z_CONST for C++ -- Clean up contrib/vstudio [Roß] -- Correct spelling error in zlib.h -- Fix mixed line endings in contrib/vstudio - -Changes in 1.2.7.3 (13 Apr 2013) -- Fix version numbers and DLL names in contrib/vstudio/*/zlib.rc - -Changes in 1.2.7.2 (13 Apr 2013) -- Change check for a four-byte type back to hexadecimal -- Fix typo in win32/Makefile.msc -- Add casts in gzwrite.c for pointer differences - -Changes in 1.2.7.1 (24 Mar 2013) -- Replace use of unsafe string functions with snprintf if available -- Avoid including stddef.h on Windows for Z_SOLO compile [Niessink] -- Fix gzgetc undefine when Z_PREFIX set [Turk] -- Eliminate use of mktemp in Makefile (not always available) -- Fix bug in 'F' mode for gzopen() -- Add inflateGetDictionary() function -- Correct comment in deflate.h -- Use _snprintf for snprintf in Microsoft C -- On Darwin, only use /usr/bin/libtool if libtool is not Apple -- Delete "--version" file if created by "ar --version" [Richard G.] -- Fix configure check for veracity of compiler error return codes -- Fix CMake compilation of static lib for MSVC2010 x64 -- Remove unused variable in infback9.c -- Fix argument checks in gzlog_compress() and gzlog_write() -- Clean up the usage of z_const and respect const usage within zlib -- Clean up examples/gzlog.[ch] comparisons of different types -- Avoid shift equal to bits in type (caused endless loop) -- Fix uninitialized value bug in gzputc() introduced by const patches -- Fix memory allocation error in examples/zran.c [Nor] -- Fix bug where gzopen(), gzclose() would write an empty file -- Fix bug in gzclose() when gzwrite() runs out of memory -- Check for input buffer malloc failure in examples/gzappend.c -- Add note to contrib/blast to use binary mode in stdio -- Fix comparisons of differently signed integers in contrib/blast -- Check for invalid code length codes in contrib/puff -- Fix serious but very rare decompression bug in inftrees.c -- Update inflateBack() comments, since inflate() can be faster -- Use underscored I/O function names for WINAPI_FAMILY -- Add _tr_flush_bits to the external symbols prefixed by --zprefix -- Add contrib/vstudio/vc10 pre-build step for static only -- Quote --version-script argument in CMakeLists.txt -- Don't specify --version-script on Apple platforms in CMakeLists.txt -- Fix casting error in contrib/testzlib/testzlib.c -- Fix types in contrib/minizip to match result of get_crc_table() -- Simplify contrib/vstudio/vc10 with 'd' suffix -- Add TOP support to win32/Makefile.msc -- Suport i686 and amd64 assembler builds in CMakeLists.txt -- Fix typos in the use of _LARGEFILE64_SOURCE in zconf.h -- Add vc11 and vc12 build files to contrib/vstudio -- Add gzvprintf() as an undocumented function in zlib -- Fix configure for Sun shell -- Remove runtime check in configure for four-byte integer type -- Add casts and consts to ease user conversion to C++ -- Add man pages for minizip and miniunzip -- In Makefile uninstall, don't rm if preceding cd fails -- Do not return Z_BUF_ERROR if deflateParam() has nothing to write - -Changes in 1.2.7 (2 May 2012) -- Replace use of memmove() with a simple copy for portability -- Test for existence of strerror -- Restore gzgetc_ for backward compatibility with 1.2.6 -- Fix build with non-GNU make on Solaris -- Require gcc 4.0 or later on Mac OS X to use the hidden attribute -- Include unistd.h for Watcom C -- Use __WATCOMC__ instead of __WATCOM__ -- Do not use the visibility attribute if NO_VIZ defined -- Improve the detection of no hidden visibility attribute -- Avoid using __int64 for gcc or solo compilation -- Cast to char * in gzprintf to avoid warnings [Zinser] -- Fix make_vms.com for VAX [Zinser] -- Don't use library or built-in byte swaps -- Simplify test and use of gcc hidden attribute -- Fix bug in gzclose_w() when gzwrite() fails to allocate memory -- Add "x" (O_EXCL) and "e" (O_CLOEXEC) modes support to gzopen() -- Fix bug in test/minigzip.c for configure --solo -- Fix contrib/vstudio project link errors [Mohanathas] -- Add ability to choose the builder in make_vms.com [Schweda] -- Add DESTDIR support to mingw32 win32/Makefile.gcc -- Fix comments in win32/Makefile.gcc for proper usage -- Allow overriding the default install locations for cmake -- Generate and install the pkg-config file with cmake -- Build both a static and a shared version of zlib with cmake -- Include version symbols for cmake builds -- If using cmake with MSVC, add the source directory to the includes -- Remove unneeded EXTRA_CFLAGS from win32/Makefile.gcc [Truta] -- Move obsolete emx makefile to old [Truta] -- Allow the use of -Wundef when compiling or using zlib -- Avoid the use of the -u option with mktemp -- Improve inflate() documentation on the use of Z_FINISH -- Recognize clang as gcc -- Add gzopen_w() in Windows for wide character path names -- Rename zconf.h in CMakeLists.txt to move it out of the way -- Add source directory in CMakeLists.txt for building examples -- Look in build directory for zlib.pc in CMakeLists.txt -- Remove gzflags from zlibvc.def in vc9 and vc10 -- Fix contrib/minizip compilation in the MinGW environment -- Update ./configure for Solaris, support --64 [Mooney] -- Remove -R. from Solaris shared build (possible security issue) -- Avoid race condition for parallel make (-j) running example -- Fix type mismatch between get_crc_table() and crc_table -- Fix parsing of version with "-" in CMakeLists.txt [Snider, Ziegler] -- Fix the path to zlib.map in CMakeLists.txt -- Force the native libtool in Mac OS X to avoid GNU libtool [Beebe] -- Add instructions to win32/Makefile.gcc for shared install [Torri] - -Changes in 1.2.6.1 (12 Feb 2012) -- Avoid the use of the Objective-C reserved name "id" -- Include io.h in gzguts.h for Microsoft compilers -- Fix problem with ./configure --prefix and gzgetc macro -- Include gz_header definition when compiling zlib solo -- Put gzflags() functionality back in zutil.c -- Avoid library header include in crc32.c for Z_SOLO -- Use name in GCC_CLASSIC as C compiler for coverage testing, if set -- Minor cleanup in contrib/minizip/zip.c [Vollant] -- Update make_vms.com [Zinser] -- Remove unnecessary gzgetc_ function -- Use optimized byte swap operations for Microsoft and GNU [Snyder] -- Fix minor typo in zlib.h comments [Rzesniowiecki] - -Changes in 1.2.6 (29 Jan 2012) -- Update the Pascal interface in contrib/pascal -- Fix function numbers for gzgetc_ in zlibvc.def files -- Fix configure.ac for contrib/minizip [Schiffer] -- Fix large-entry detection in minizip on 64-bit systems [Schiffer] -- Have ./configure use the compiler return code for error indication -- Fix CMakeLists.txt for cross compilation [McClure] -- Fix contrib/minizip/zip.c for 64-bit architectures [Dalsnes] -- Fix compilation of contrib/minizip on FreeBSD [Marquez] -- Correct suggested usages in win32/Makefile.msc [Shachar, Horvath] -- Include io.h for Turbo C / Borland C on all platforms [Truta] -- Make version explicit in contrib/minizip/configure.ac [Bosmans] -- Avoid warning for no encryption in contrib/minizip/zip.c [Vollant] -- Minor cleanup up contrib/minizip/unzip.c [Vollant] -- Fix bug when compiling minizip with C++ [Vollant] -- Protect for long name and extra fields in contrib/minizip [Vollant] -- Avoid some warnings in contrib/minizip [Vollant] -- Add -I../.. -L../.. to CFLAGS for minizip and miniunzip -- Add missing libs to minizip linker command -- Add support for VPATH builds in contrib/minizip -- Add an --enable-demos option to contrib/minizip/configure -- Add the generation of configure.log by ./configure -- Exit when required parameters not provided to win32/Makefile.gcc -- Have gzputc return the character written instead of the argument -- Use the -m option on ldconfig for BSD systems [Tobias] -- Correct in zlib.map when deflateResetKeep was added - -Changes in 1.2.5.3 (15 Jan 2012) -- Restore gzgetc function for binary compatibility -- Do not use _lseeki64 under Borland C++ [Truta] -- Update win32/Makefile.msc to build test/*.c [Truta] -- Remove old/visualc6 given CMakefile and other alternatives -- Update AS400 build files and documentation [Monnerat] -- Update win32/Makefile.gcc to build test/*.c [Truta] -- Permit stronger flushes after Z_BLOCK flushes -- Avoid extraneous empty blocks when doing empty flushes -- Permit Z_NULL arguments to deflatePending -- Allow deflatePrime() to insert bits in the middle of a stream -- Remove second empty static block for Z_PARTIAL_FLUSH -- Write out all of the available bits when using Z_BLOCK -- Insert the first two strings in the hash table after a flush - -Changes in 1.2.5.2 (17 Dec 2011) -- fix ld error: unable to find version dependency 'ZLIB_1.2.5' -- use relative symlinks for shared libs -- Avoid searching past window for Z_RLE strategy -- Assure that high-water mark initialization is always applied in deflate -- Add assertions to fill_window() in deflate.c to match comments -- Update python link in README -- Correct spelling error in gzread.c -- Fix bug in gzgets() for a concatenated empty gzip stream -- Correct error in comment for gz_make() -- Change gzread() and related to ignore junk after gzip streams -- Allow gzread() and related to continue after gzclearerr() -- Allow gzrewind() and gzseek() after a premature end-of-file -- Simplify gzseek() now that raw after gzip is ignored -- Change gzgetc() to a macro for speed (~40% speedup in testing) -- Fix gzclose() to return the actual error last encountered -- Always add large file support for windows -- Include zconf.h for windows large file support -- Include zconf.h.cmakein for windows large file support -- Update zconf.h.cmakein on make distclean -- Merge vestigial vsnprintf determination from zutil.h to gzguts.h -- Clarify how gzopen() appends in zlib.h comments -- Correct documentation of gzdirect() since junk at end now ignored -- Add a transparent write mode to gzopen() when 'T' is in the mode -- Update python link in zlib man page -- Get inffixed.h and MAKEFIXED result to match -- Add a ./config --solo option to make zlib subset with no library use -- Add undocumented inflateResetKeep() function for CAB file decoding -- Add --cover option to ./configure for gcc coverage testing -- Add #define ZLIB_CONST option to use const in the z_stream interface -- Add comment to gzdopen() in zlib.h to use dup() when using fileno() -- Note behavior of uncompress() to provide as much data as it can -- Add files in contrib/minizip to aid in building libminizip -- Split off AR options in Makefile.in and configure -- Change ON macro to Z_ARG to avoid application conflicts -- Facilitate compilation with Borland C++ for pragmas and vsnprintf -- Include io.h for Turbo C / Borland C++ -- Move example.c and minigzip.c to test/ -- Simplify incomplete code table filling in inflate_table() -- Remove code from inflate.c and infback.c that is impossible to execute -- Test the inflate code with full coverage -- Allow deflateSetDictionary, inflateSetDictionary at any time (in raw) -- Add deflateResetKeep and fix inflateResetKeep to retain dictionary -- Fix gzwrite.c to accommodate reduced memory zlib compilation -- Have inflate() with Z_FINISH avoid the allocation of a window -- Do not set strm->adler when doing raw inflate -- Fix gzeof() to behave just like feof() when read is not past end of file -- Fix bug in gzread.c when end-of-file is reached -- Avoid use of Z_BUF_ERROR in gz* functions except for premature EOF -- Document gzread() capability to read concurrently written files -- Remove hard-coding of resource compiler in CMakeLists.txt [Blammo] - -Changes in 1.2.5.1 (10 Sep 2011) -- Update FAQ entry on shared builds (#13) -- Avoid symbolic argument to chmod in Makefile.in -- Fix bug and add consts in contrib/puff [Oberhumer] -- Update contrib/puff/zeros.raw test file to have all block types -- Add full coverage test for puff in contrib/puff/Makefile -- Fix static-only-build install in Makefile.in -- Fix bug in unzGetCurrentFileInfo() in contrib/minizip [Kuno] -- Add libz.a dependency to shared in Makefile.in for parallel builds -- Spell out "number" (instead of "nb") in zlib.h for total_in, total_out -- Replace $(...) with `...` in configure for non-bash sh [Bowler] -- Add darwin* to Darwin* and solaris* to SunOS\ 5* in configure [Groffen] -- Add solaris* to Linux* in configure to allow gcc use [Groffen] -- Add *bsd* to Linux* case in configure [Bar-Lev] -- Add inffast.obj to dependencies in win32/Makefile.msc -- Correct spelling error in deflate.h [Kohler] -- Change libzdll.a again to libz.dll.a (!) in win32/Makefile.gcc -- Add test to configure for GNU C looking for gcc in output of $cc -v -- Add zlib.pc generation to win32/Makefile.gcc [Weigelt] -- Fix bug in zlib.h for _FILE_OFFSET_BITS set and _LARGEFILE64_SOURCE not -- Add comment in zlib.h that adler32_combine with len2 < 0 makes no sense -- Make NO_DIVIDE option in adler32.c much faster (thanks to John Reiser) -- Make stronger test in zconf.h to include unistd.h for LFS -- Apply Darwin patches for 64-bit file offsets to contrib/minizip [Slack] -- Fix zlib.h LFS support when Z_PREFIX used -- Add updated as400 support (removed from old) [Monnerat] -- Avoid deflate sensitivity to volatile input data -- Avoid division in adler32_combine for NO_DIVIDE -- Clarify the use of Z_FINISH with deflateBound() amount of space -- Set binary for output file in puff.c -- Use u4 type for crc_table to avoid conversion warnings -- Apply casts in zlib.h to avoid conversion warnings -- Add OF to prototypes for adler32_combine_ and crc32_combine_ [Miller] -- Improve inflateSync() documentation to note indeterminancy -- Add deflatePending() function to return the amount of pending output -- Correct the spelling of "specification" in FAQ [Randers-Pehrson] -- Add a check in configure for stdarg.h, use for gzprintf() -- Check that pointers fit in ints when gzprint() compiled old style -- Add dummy name before $(SHAREDLIBV) in Makefile [Bar-Lev, Bowler] -- Delete line in configure that adds -L. libz.a to LDFLAGS [Weigelt] -- Add debug records in assmebler code [Londer] -- Update RFC references to use http://tools.ietf.org/html/... [Li] -- Add --archs option, use of libtool to configure for Mac OS X [Borstel] - -Changes in 1.2.5 (19 Apr 2010) -- Disable visibility attribute in win32/Makefile.gcc [Bar-Lev] -- Default to libdir as sharedlibdir in configure [Nieder] -- Update copyright dates on modified source files -- Update trees.c to be able to generate modified trees.h -- Exit configure for MinGW, suggesting win32/Makefile.gcc -- Check for NULL path in gz_open [Homurlu] - -Changes in 1.2.4.5 (18 Apr 2010) -- Set sharedlibdir in configure [Torok] -- Set LDFLAGS in Makefile.in [Bar-Lev] -- Avoid mkdir objs race condition in Makefile.in [Bowler] -- Add ZLIB_INTERNAL in front of internal inter-module functions and arrays -- Define ZLIB_INTERNAL to hide internal functions and arrays for GNU C -- Don't use hidden attribute when it is a warning generator (e.g. Solaris) - -Changes in 1.2.4.4 (18 Apr 2010) -- Fix CROSS_PREFIX executable testing, CHOST extract, mingw* [Torok] -- Undefine _LARGEFILE64_SOURCE in zconf.h if it is zero, but not if empty -- Try to use bash or ksh regardless of functionality of /bin/sh -- Fix configure incompatibility with NetBSD sh -- Remove attempt to run under bash or ksh since have better NetBSD fix -- Fix win32/Makefile.gcc for MinGW [Bar-Lev] -- Add diagnostic messages when using CROSS_PREFIX in configure -- Added --sharedlibdir option to configure [Weigelt] -- Use hidden visibility attribute when available [Frysinger] - -Changes in 1.2.4.3 (10 Apr 2010) -- Only use CROSS_PREFIX in configure for ar and ranlib if they exist -- Use CROSS_PREFIX for nm [Bar-Lev] -- Assume _LARGEFILE64_SOURCE defined is equivalent to true -- Avoid use of undefined symbols in #if with && and || -- Make *64 prototypes in gzguts.h consistent with functions -- Add -shared load option for MinGW in configure [Bowler] -- Move z_off64_t to public interface, use instead of off64_t -- Remove ! from shell test in configure (not portable to Solaris) -- Change +0 macro tests to -0 for possibly increased portability - -Changes in 1.2.4.2 (9 Apr 2010) -- Add consistent carriage returns to readme.txt's in masmx86 and masmx64 -- Really provide prototypes for *64 functions when building without LFS -- Only define unlink() in minigzip.c if unistd.h not included -- Update README to point to contrib/vstudio project files -- Move projects/vc6 to old/ and remove projects/ -- Include stdlib.h in minigzip.c for setmode() definition under WinCE -- Clean up assembler builds in win32/Makefile.msc [Rowe] -- Include sys/types.h for Microsoft for off_t definition -- Fix memory leak on error in gz_open() -- Symbolize nm as $NM in configure [Weigelt] -- Use TEST_LDSHARED instead of LDSHARED to link test programs [Weigelt] -- Add +0 to _FILE_OFFSET_BITS and _LFS64_LARGEFILE in case not defined -- Fix bug in gzeof() to take into account unused input data -- Avoid initialization of structures with variables in puff.c -- Updated win32/README-WIN32.txt [Rowe] - -Changes in 1.2.4.1 (28 Mar 2010) -- Remove the use of [a-z] constructs for sed in configure [gentoo 310225] -- Remove $(SHAREDLIB) from LIBS in Makefile.in [Creech] -- Restore "for debugging" comment on sprintf() in gzlib.c -- Remove fdopen for MVS from gzguts.h -- Put new README-WIN32.txt in win32 [Rowe] -- Add check for shell to configure and invoke another shell if needed -- Fix big fat stinking bug in gzseek() on uncompressed files -- Remove vestigial F_OPEN64 define in zutil.h -- Set and check the value of _LARGEFILE_SOURCE and _LARGEFILE64_SOURCE -- Avoid errors on non-LFS systems when applications define LFS macros -- Set EXE to ".exe" in configure for MINGW [Kahle] -- Match crc32() in crc32.c exactly to the prototype in zlib.h [Sherrill] -- Add prefix for cross-compilation in win32/makefile.gcc [Bar-Lev] -- Add DLL install in win32/makefile.gcc [Bar-Lev] -- Allow Linux* or linux* from uname in configure [Bar-Lev] -- Allow ldconfig to be redefined in configure and Makefile.in [Bar-Lev] -- Add cross-compilation prefixes to configure [Bar-Lev] -- Match type exactly in gz_load() invocation in gzread.c -- Match type exactly of zcalloc() in zutil.c to zlib.h alloc_func -- Provide prototypes for *64 functions when building zlib without LFS -- Don't use -lc when linking shared library on MinGW -- Remove errno.h check in configure and vestigial errno code in zutil.h - -Changes in 1.2.4 (14 Mar 2010) -- Fix VER3 extraction in configure for no fourth subversion -- Update zlib.3, add docs to Makefile.in to make .pdf out of it -- Add zlib.3.pdf to distribution -- Don't set error code in gzerror() if passed pointer is NULL -- Apply destination directory fixes to CMakeLists.txt [Lowman] -- Move #cmakedefine's to a new zconf.in.cmakein -- Restore zconf.h for builds that don't use configure or cmake -- Add distclean to dummy Makefile for convenience -- Update and improve INDEX, README, and FAQ -- Update CMakeLists.txt for the return of zconf.h [Lowman] -- Update contrib/vstudio/vc9 and vc10 [Vollant] -- Change libz.dll.a back to libzdll.a in win32/Makefile.gcc -- Apply license and readme changes to contrib/asm686 [Raiter] -- Check file name lengths and add -c option in minigzip.c [Li] -- Update contrib/amd64 and contrib/masmx86/ [Vollant] -- Avoid use of "eof" parameter in trees.c to not shadow library variable -- Update make_vms.com for removal of zlibdefs.h [Zinser] -- Update assembler code and vstudio projects in contrib [Vollant] -- Remove outdated assembler code contrib/masm686 and contrib/asm586 -- Remove old vc7 and vc8 from contrib/vstudio -- Update win32/Makefile.msc, add ZLIB_VER_SUBREVISION [Rowe] -- Fix memory leaks in gzclose_r() and gzclose_w(), file leak in gz_open() -- Add contrib/gcc_gvmat64 for longest_match and inflate_fast [Vollant] -- Remove *64 functions from win32/zlib.def (they're not 64-bit yet) -- Fix bug in void-returning vsprintf() case in gzwrite.c -- Fix name change from inflate.h in contrib/inflate86/inffas86.c -- Check if temporary file exists before removing in make_vms.com [Zinser] -- Fix make install and uninstall for --static option -- Fix usage of _MSC_VER in gzguts.h and zutil.h [Truta] -- Update readme.txt in contrib/masmx64 and masmx86 to assemble - -Changes in 1.2.3.9 (21 Feb 2010) -- Expunge gzio.c -- Move as400 build information to old -- Fix updates in contrib/minizip and contrib/vstudio -- Add const to vsnprintf test in configure to avoid warnings [Weigelt] -- Delete zconf.h (made by configure) [Weigelt] -- Change zconf.in.h to zconf.h.in per convention [Weigelt] -- Check for NULL buf in gzgets() -- Return empty string for gzgets() with len == 1 (like fgets()) -- Fix description of gzgets() in zlib.h for end-of-file, NULL return -- Update minizip to 1.1 [Vollant] -- Avoid MSVC loss of data warnings in gzread.c, gzwrite.c -- Note in zlib.h that gzerror() should be used to distinguish from EOF -- Remove use of snprintf() from gzlib.c -- Fix bug in gzseek() -- Update contrib/vstudio, adding vc9 and vc10 [Kuno, Vollant] -- Fix zconf.h generation in CMakeLists.txt [Lowman] -- Improve comments in zconf.h where modified by configure - -Changes in 1.2.3.8 (13 Feb 2010) -- Clean up text files (tabs, trailing whitespace, etc.) [Oberhumer] -- Use z_off64_t in gz_zero() and gz_skip() to match state->skip -- Avoid comparison problem when sizeof(int) == sizeof(z_off64_t) -- Revert to Makefile.in from 1.2.3.6 (live with the clutter) -- Fix missing error return in gzflush(), add zlib.h note -- Add *64 functions to zlib.map [Levin] -- Fix signed/unsigned comparison in gz_comp() -- Use SFLAGS when testing shared linking in configure -- Add --64 option to ./configure to use -m64 with gcc -- Fix ./configure --help to correctly name options -- Have make fail if a test fails [Levin] -- Avoid buffer overrun in contrib/masmx64/gvmat64.asm [Simpson] -- Remove assembler object files from contrib - -Changes in 1.2.3.7 (24 Jan 2010) -- Always gzopen() with O_LARGEFILE if available -- Fix gzdirect() to work immediately after gzopen() or gzdopen() -- Make gzdirect() more precise when the state changes while reading -- Improve zlib.h documentation in many places -- Catch memory allocation failure in gz_open() -- Complete close operation if seek forward in gzclose_w() fails -- Return Z_ERRNO from gzclose_r() if close() fails -- Return Z_STREAM_ERROR instead of EOF for gzclose() being passed NULL -- Return zero for gzwrite() errors to match zlib.h description -- Return -1 on gzputs() error to match zlib.h description -- Add zconf.in.h to allow recovery from configure modification [Weigelt] -- Fix static library permissions in Makefile.in [Weigelt] -- Avoid warnings in configure tests that hide functionality [Weigelt] -- Add *BSD and DragonFly to Linux case in configure [gentoo 123571] -- Change libzdll.a to libz.dll.a in win32/Makefile.gcc [gentoo 288212] -- Avoid access of uninitialized data for first inflateReset2 call [Gomes] -- Keep object files in subdirectories to reduce the clutter somewhat -- Remove default Makefile and zlibdefs.h, add dummy Makefile -- Add new external functions to Z_PREFIX, remove duplicates, z_z_ -> z_ -- Remove zlibdefs.h completely -- modify zconf.h instead - -Changes in 1.2.3.6 (17 Jan 2010) -- Avoid void * arithmetic in gzread.c and gzwrite.c -- Make compilers happier with const char * for gz_error message -- Avoid unused parameter warning in inflate.c -- Avoid signed-unsigned comparison warning in inflate.c -- Indent #pragma's for traditional C -- Fix usage of strwinerror() in glib.c, change to gz_strwinerror() -- Correct email address in configure for system options -- Update make_vms.com and add make_vms.com to contrib/minizip [Zinser] -- Update zlib.map [Brown] -- Fix Makefile.in for Solaris 10 make of example64 and minizip64 [Torok] -- Apply various fixes to CMakeLists.txt [Lowman] -- Add checks on len in gzread() and gzwrite() -- Add error message for no more room for gzungetc() -- Remove zlib version check in gzwrite() -- Defer compression of gzprintf() result until need to -- Use snprintf() in gzdopen() if available -- Remove USE_MMAP configuration determination (only used by minigzip) -- Remove examples/pigz.c (available separately) -- Update examples/gun.c to 1.6 - -Changes in 1.2.3.5 (8 Jan 2010) -- Add space after #if in zutil.h for some compilers -- Fix relatively harmless bug in deflate_fast() [Exarevsky] -- Fix same problem in deflate_slow() -- Add $(SHAREDLIBV) to LIBS in Makefile.in [Brown] -- Add deflate_rle() for faster Z_RLE strategy run-length encoding -- Add deflate_huff() for faster Z_HUFFMAN_ONLY encoding -- Change name of "write" variable in inffast.c to avoid library collisions -- Fix premature EOF from gzread() in gzio.c [Brown] -- Use zlib header window size if windowBits is 0 in inflateInit2() -- Remove compressBound() call in deflate.c to avoid linking compress.o -- Replace use of errno in gz* with functions, support WinCE [Alves] -- Provide alternative to perror() in minigzip.c for WinCE [Alves] -- Don't use _vsnprintf on later versions of MSVC [Lowman] -- Add CMake build script and input file [Lowman] -- Update contrib/minizip to 1.1 [Svensson, Vollant] -- Moved nintendods directory from contrib to . -- Replace gzio.c with a new set of routines with the same functionality -- Add gzbuffer(), gzoffset(), gzclose_r(), gzclose_w() as part of above -- Update contrib/minizip to 1.1b -- Change gzeof() to return 0 on error instead of -1 to agree with zlib.h - -Changes in 1.2.3.4 (21 Dec 2009) -- Use old school .SUFFIXES in Makefile.in for FreeBSD compatibility -- Update comments in configure and Makefile.in for default --shared -- Fix test -z's in configure [Marquess] -- Build examplesh and minigzipsh when not testing -- Change NULL's to Z_NULL's in deflate.c and in comments in zlib.h -- Import LDFLAGS from the environment in configure -- Fix configure to populate SFLAGS with discovered CFLAGS options -- Adapt make_vms.com to the new Makefile.in [Zinser] -- Add zlib2ansi script for C++ compilation [Marquess] -- Add _FILE_OFFSET_BITS=64 test to make test (when applicable) -- Add AMD64 assembler code for longest match to contrib [Teterin] -- Include options from $SFLAGS when doing $LDSHARED -- Simplify 64-bit file support by introducing z_off64_t type -- Make shared object files in objs directory to work around old Sun cc -- Use only three-part version number for Darwin shared compiles -- Add rc option to ar in Makefile.in for when ./configure not run -- Add -WI,-rpath,. to LDFLAGS for OSF 1 V4* -- Set LD_LIBRARYN32_PATH for SGI IRIX shared compile -- Protect against _FILE_OFFSET_BITS being defined when compiling zlib -- Rename Makefile.in targets allstatic to static and allshared to shared -- Fix static and shared Makefile.in targets to be independent -- Correct error return bug in gz_open() by setting state [Brown] -- Put spaces before ;;'s in configure for better sh compatibility -- Add pigz.c (parallel implementation of gzip) to examples/ -- Correct constant in crc32.c to UL [Leventhal] -- Reject negative lengths in crc32_combine() -- Add inflateReset2() function to work like inflateEnd()/inflateInit2() -- Include sys/types.h for _LARGEFILE64_SOURCE [Brown] -- Correct typo in doc/algorithm.txt [Janik] -- Fix bug in adler32_combine() [Zhu] -- Catch missing-end-of-block-code error in all inflates and in puff - Assures that random input to inflate eventually results in an error -- Added enough.c (calculation of ENOUGH for inftrees.h) to examples/ -- Update ENOUGH and its usage to reflect discovered bounds -- Fix gzerror() error report on empty input file [Brown] -- Add ush casts in trees.c to avoid pedantic runtime errors -- Fix typo in zlib.h uncompress() description [Reiss] -- Correct inflate() comments with regard to automatic header detection -- Remove deprecation comment on Z_PARTIAL_FLUSH (it stays) -- Put new version of gzlog (2.0) in examples with interruption recovery -- Add puff compile option to permit invalid distance-too-far streams -- Add puff TEST command options, ability to read piped input -- Prototype the *64 functions in zlib.h when _FILE_OFFSET_BITS == 64, but - _LARGEFILE64_SOURCE not defined -- Fix Z_FULL_FLUSH to truly erase the past by resetting s->strstart -- Fix deflateSetDictionary() to use all 32K for output consistency -- Remove extraneous #define MIN_LOOKAHEAD in deflate.c (in deflate.h) -- Clear bytes after deflate lookahead to avoid use of uninitialized data -- Change a limit in inftrees.c to be more transparent to Coverity Prevent -- Update win32/zlib.def with exported symbols from zlib.h -- Correct spelling errors in zlib.h [Willem, Sobrado] -- Allow Z_BLOCK for deflate() to force a new block -- Allow negative bits in inflatePrime() to delete existing bit buffer -- Add Z_TREES flush option to inflate() to return at end of trees -- Add inflateMark() to return current state information for random access -- Add Makefile for NintendoDS to contrib [Costa] -- Add -w in configure compile tests to avoid spurious warnings [Beucler] -- Fix typos in zlib.h comments for deflateSetDictionary() -- Fix EOF detection in transparent gzread() [Maier] - -Changes in 1.2.3.3 (2 October 2006) -- Make --shared the default for configure, add a --static option -- Add compile option to permit invalid distance-too-far streams -- Add inflateUndermine() function which is required to enable above -- Remove use of "this" variable name for C++ compatibility [Marquess] -- Add testing of shared library in make test, if shared library built -- Use ftello() and fseeko() if available instead of ftell() and fseek() -- Provide two versions of all functions that use the z_off_t type for - binary compatibility -- a normal version and a 64-bit offset version, - per the Large File Support Extension when _LARGEFILE64_SOURCE is - defined; use the 64-bit versions by default when _FILE_OFFSET_BITS - is defined to be 64 -- Add a --uname= option to configure to perhaps help with cross-compiling - -Changes in 1.2.3.2 (3 September 2006) -- Turn off silly Borland warnings [Hay] -- Use off64_t and define _LARGEFILE64_SOURCE when present -- Fix missing dependency on inffixed.h in Makefile.in -- Rig configure --shared to build both shared and static [Teredesai, Truta] -- Remove zconf.in.h and instead create a new zlibdefs.h file -- Fix contrib/minizip/unzip.c non-encrypted after encrypted [Vollant] -- Add treebuild.xml (see http://treebuild.metux.de/) [Weigelt] - -Changes in 1.2.3.1 (16 August 2006) -- Add watcom directory with OpenWatcom make files [Daniel] -- Remove #undef of FAR in zconf.in.h for MVS [Fedtke] -- Update make_vms.com [Zinser] -- Use -fPIC for shared build in configure [Teredesai, Nicholson] -- Use only major version number for libz.so on IRIX and OSF1 [Reinholdtsen] -- Use fdopen() (not _fdopen()) for Interix in zutil.h [Bäck] -- Add some FAQ entries about the contrib directory -- Update the MVS question in the FAQ -- Avoid extraneous reads after EOF in gzio.c [Brown] -- Correct spelling of "successfully" in gzio.c [Randers-Pehrson] -- Add comments to zlib.h about gzerror() usage [Brown] -- Set extra flags in gzip header in gzopen() like deflate() does -- Make configure options more compatible with double-dash conventions - [Weigelt] -- Clean up compilation under Solaris SunStudio cc [Rowe, Reinholdtsen] -- Fix uninstall target in Makefile.in [Truta] -- Add pkgconfig support [Weigelt] -- Use $(DESTDIR) macro in Makefile.in [Reinholdtsen, Weigelt] -- Replace set_data_type() with a more accurate detect_data_type() in - trees.c, according to the txtvsbin.txt document [Truta] -- Swap the order of #include and #include "zlib.h" in - gzio.c, example.c and minigzip.c [Truta] -- Shut up annoying VS2005 warnings about standard C deprecation [Rowe, - Truta] (where?) -- Fix target "clean" from win32/Makefile.bor [Truta] -- Create .pdb and .manifest files in win32/makefile.msc [Ziegler, Rowe] -- Update zlib www home address in win32/DLL_FAQ.txt [Truta] -- Update contrib/masmx86/inffas32.asm for VS2005 [Vollant, Van Wassenhove] -- Enable browse info in the "Debug" and "ASM Debug" configurations in - the Visual C++ 6 project, and set (non-ASM) "Debug" as default [Truta] -- Add pkgconfig support [Weigelt] -- Add ZLIB_VER_MAJOR, ZLIB_VER_MINOR and ZLIB_VER_REVISION in zlib.h, - for use in win32/zlib1.rc [Polushin, Rowe, Truta] -- Add a document that explains the new text detection scheme to - doc/txtvsbin.txt [Truta] -- Add rfc1950.txt, rfc1951.txt and rfc1952.txt to doc/ [Truta] -- Move algorithm.txt into doc/ [Truta] -- Synchronize FAQ with website -- Fix compressBound(), was low for some pathological cases [Fearnley] -- Take into account wrapper variations in deflateBound() -- Set examples/zpipe.c input and output to binary mode for Windows -- Update examples/zlib_how.html with new zpipe.c (also web site) -- Fix some warnings in examples/gzlog.c and examples/zran.c (it seems - that gcc became pickier in 4.0) -- Add zlib.map for Linux: "All symbols from zlib-1.1.4 remain - un-versioned, the patch adds versioning only for symbols introduced in - zlib-1.2.0 or later. It also declares as local those symbols which are - not designed to be exported." [Levin] -- Update Z_PREFIX list in zconf.in.h, add --zprefix option to configure -- Do not initialize global static by default in trees.c, add a response - NO_INIT_GLOBAL_POINTERS to initialize them if needed [Marquess] -- Don't use strerror() in gzio.c under WinCE [Yakimov] -- Don't use errno.h in zutil.h under WinCE [Yakimov] -- Move arguments for AR to its usage to allow replacing ar [Marot] -- Add HAVE_VISIBILITY_PRAGMA in zconf.in.h for Mozilla [Randers-Pehrson] -- Improve inflateInit() and inflateInit2() documentation -- Fix structure size comment in inflate.h -- Change configure help option from --h* to --help [Santos] - -Changes in 1.2.3 (18 July 2005) -- Apply security vulnerability fixes to contrib/infback9 as well -- Clean up some text files (carriage returns, trailing space) -- Update testzlib, vstudio, masmx64, and masmx86 in contrib [Vollant] - -Changes in 1.2.2.4 (11 July 2005) -- Add inflatePrime() function for starting inflation at bit boundary -- Avoid some Visual C warnings in deflate.c -- Avoid more silly Visual C warnings in inflate.c and inftrees.c for 64-bit - compile -- Fix some spelling errors in comments [Betts] -- Correct inflateInit2() error return documentation in zlib.h -- Add zran.c example of compressed data random access to examples - directory, shows use of inflatePrime() -- Fix cast for assignments to strm->state in inflate.c and infback.c -- Fix zlibCompileFlags() in zutil.c to use 1L for long shifts [Oberhumer] -- Move declarations of gf2 functions to right place in crc32.c [Oberhumer] -- Add cast in trees.c t avoid a warning [Oberhumer] -- Avoid some warnings in fitblk.c, gun.c, gzjoin.c in examples [Oberhumer] -- Update make_vms.com [Zinser] -- Initialize state->write in inflateReset() since copied in inflate_fast() -- Be more strict on incomplete code sets in inflate_table() and increase - ENOUGH and MAXD -- this repairs a possible security vulnerability for - invalid inflate input. Thanks to Tavis Ormandy and Markus Oberhumer for - discovering the vulnerability and providing test cases. -- Add ia64 support to configure for HP-UX [Smith] -- Add error return to gzread() for format or i/o error [Levin] -- Use malloc.h for OS/2 [Necasek] - -Changes in 1.2.2.3 (27 May 2005) -- Replace 1U constants in inflate.c and inftrees.c for 64-bit compile -- Typecast fread() return values in gzio.c [Vollant] -- Remove trailing space in minigzip.c outmode (VC++ can't deal with it) -- Fix crc check bug in gzread() after gzungetc() [Heiner] -- Add the deflateTune() function to adjust internal compression parameters -- Add a fast gzip decompressor, gun.c, to examples (use of inflateBack) -- Remove an incorrect assertion in examples/zpipe.c -- Add C++ wrapper in infback9.h [Donais] -- Fix bug in inflateCopy() when decoding fixed codes -- Note in zlib.h how much deflateSetDictionary() actually uses -- Remove USE_DICT_HEAD in deflate.c (would mess up inflate if used) -- Add _WIN32_WCE to define WIN32 in zconf.in.h [Spencer] -- Don't include stderr.h or errno.h for _WIN32_WCE in zutil.h [Spencer] -- Add gzdirect() function to indicate transparent reads -- Update contrib/minizip [Vollant] -- Fix compilation of deflate.c when both ASMV and FASTEST [Oberhumer] -- Add casts in crc32.c to avoid warnings [Oberhumer] -- Add contrib/masmx64 [Vollant] -- Update contrib/asm586, asm686, masmx86, testzlib, vstudio [Vollant] - -Changes in 1.2.2.2 (30 December 2004) -- Replace structure assignments in deflate.c and inflate.c with zmemcpy to - avoid implicit memcpy calls (portability for no-library compilation) -- Increase sprintf() buffer size in gzdopen() to allow for large numbers -- Add INFLATE_STRICT to check distances against zlib header -- Improve WinCE errno handling and comments [Chang] -- Remove comment about no gzip header processing in FAQ -- Add Z_FIXED strategy option to deflateInit2() to force fixed trees -- Add updated make_vms.com [Coghlan], update README -- Create a new "examples" directory, move gzappend.c there, add zpipe.c, - fitblk.c, gzlog.[ch], gzjoin.c, and zlib_how.html. -- Add FAQ entry and comments in deflate.c on uninitialized memory access -- Add Solaris 9 make options in configure [Gilbert] -- Allow strerror() usage in gzio.c for STDC -- Fix DecompressBuf in contrib/delphi/ZLib.pas [ManChesTer] -- Update contrib/masmx86/inffas32.asm and gvmat32.asm [Vollant] -- Use z_off_t for adler32_combine() and crc32_combine() lengths -- Make adler32() much faster for small len -- Use OS_CODE in deflate() default gzip header - -Changes in 1.2.2.1 (31 October 2004) -- Allow inflateSetDictionary() call for raw inflate -- Fix inflate header crc check bug for file names and comments -- Add deflateSetHeader() and gz_header structure for custom gzip headers -- Add inflateGetheader() to retrieve gzip headers -- Add crc32_combine() and adler32_combine() functions -- Add alloc_func, free_func, in_func, out_func to Z_PREFIX list -- Use zstreamp consistently in zlib.h (inflate_back functions) -- Remove GUNZIP condition from definition of inflate_mode in inflate.h - and in contrib/inflate86/inffast.S [Truta, Anderson] -- Add support for AMD64 in contrib/inflate86/inffas86.c [Anderson] -- Update projects/README.projects and projects/visualc6 [Truta] -- Update win32/DLL_FAQ.txt [Truta] -- Avoid warning under NO_GZCOMPRESS in gzio.c; fix typo [Truta] -- Deprecate Z_ASCII; use Z_TEXT instead [Truta] -- Use a new algorithm for setting strm->data_type in trees.c [Truta] -- Do not define an exit() prototype in zutil.c unless DEBUG defined -- Remove prototype of exit() from zutil.c, example.c, minigzip.c [Truta] -- Add comment in zlib.h for Z_NO_FLUSH parameter to deflate() -- Fix Darwin build version identification [Peterson] - -Changes in 1.2.2 (3 October 2004) -- Update zlib.h comments on gzip in-memory processing -- Set adler to 1 in inflateReset() to support Java test suite [Walles] -- Add contrib/dotzlib [Ravn] -- Update win32/DLL_FAQ.txt [Truta] -- Update contrib/minizip [Vollant] -- Move contrib/visual-basic.txt to old/ [Truta] -- Fix assembler builds in projects/visualc6/ [Truta] - -Changes in 1.2.1.2 (9 September 2004) -- Update INDEX file -- Fix trees.c to update strm->data_type (no one ever noticed!) -- Fix bug in error case in inflate.c, infback.c, and infback9.c [Brown] -- Add "volatile" to crc table flag declaration (for DYNAMIC_CRC_TABLE) -- Add limited multitasking protection to DYNAMIC_CRC_TABLE -- Add NO_vsnprintf for VMS in zutil.h [Mozilla] -- Don't declare strerror() under VMS [Mozilla] -- Add comment to DYNAMIC_CRC_TABLE to use get_crc_table() to initialize -- Update contrib/ada [Anisimkov] -- Update contrib/minizip [Vollant] -- Fix configure to not hardcode directories for Darwin [Peterson] -- Fix gzio.c to not return error on empty files [Brown] -- Fix indentation; update version in contrib/delphi/ZLib.pas and - contrib/pascal/zlibpas.pas [Truta] -- Update mkasm.bat in contrib/masmx86 [Truta] -- Update contrib/untgz [Truta] -- Add projects/README.projects [Truta] -- Add project for MS Visual C++ 6.0 in projects/visualc6 [Cadieux, Truta] -- Update win32/DLL_FAQ.txt [Truta] -- Update list of Z_PREFIX symbols in zconf.h [Randers-Pehrson, Truta] -- Remove an unnecessary assignment to curr in inftrees.c [Truta] -- Add OS/2 to exe builds in configure [Poltorak] -- Remove err dummy parameter in zlib.h [Kientzle] - -Changes in 1.2.1.1 (9 January 2004) -- Update email address in README -- Several FAQ updates -- Fix a big fat bug in inftrees.c that prevented decoding valid - dynamic blocks with only literals and no distance codes -- - Thanks to "Hot Emu" for the bug report and sample file -- Add a note to puff.c on no distance codes case. - -Changes in 1.2.1 (17 November 2003) -- Remove a tab in contrib/gzappend/gzappend.c -- Update some interfaces in contrib for new zlib functions -- Update zlib version number in some contrib entries -- Add Windows CE definition for ptrdiff_t in zutil.h [Mai, Truta] -- Support shared libraries on Hurd and KFreeBSD [Brown] -- Fix error in NO_DIVIDE option of adler32.c - -Changes in 1.2.0.8 (4 November 2003) -- Update version in contrib/delphi/ZLib.pas and contrib/pascal/zlibpas.pas -- Add experimental NO_DIVIDE #define in adler32.c - - Possibly faster on some processors (let me know if it is) -- Correct Z_BLOCK to not return on first inflate call if no wrap -- Fix strm->data_type on inflate() return to correctly indicate EOB -- Add deflatePrime() function for appending in the middle of a byte -- Add contrib/gzappend for an example of appending to a stream -- Update win32/DLL_FAQ.txt [Truta] -- Delete Turbo C comment in README [Truta] -- Improve some indentation in zconf.h [Truta] -- Fix infinite loop on bad input in configure script [Church] -- Fix gzeof() for concatenated gzip files [Johnson] -- Add example to contrib/visual-basic.txt [Michael B.] -- Add -p to mkdir's in Makefile.in [vda] -- Fix configure to properly detect presence or lack of printf functions -- Add AS400 support [Monnerat] -- Add a little Cygwin support [Wilson] - -Changes in 1.2.0.7 (21 September 2003) -- Correct some debug formats in contrib/infback9 -- Cast a type in a debug statement in trees.c -- Change search and replace delimiter in configure from % to # [Beebe] -- Update contrib/untgz to 0.2 with various fixes [Truta] -- Add build support for Amiga [Nikl] -- Remove some directories in old that have been updated to 1.2 -- Add dylib building for Mac OS X in configure and Makefile.in -- Remove old distribution stuff from Makefile -- Update README to point to DLL_FAQ.txt, and add comment on Mac OS X -- Update links in README - -Changes in 1.2.0.6 (13 September 2003) -- Minor FAQ updates -- Update contrib/minizip to 1.00 [Vollant] -- Remove test of gz functions in example.c when GZ_COMPRESS defined [Truta] -- Update POSTINC comment for 68060 [Nikl] -- Add contrib/infback9 with deflate64 decoding (unsupported) -- For MVS define NO_vsnprintf and undefine FAR [van Burik] -- Add pragma for fdopen on MVS [van Burik] - -Changes in 1.2.0.5 (8 September 2003) -- Add OF to inflateBackEnd() declaration in zlib.h -- Remember start when using gzdopen in the middle of a file -- Use internal off_t counters in gz* functions to properly handle seeks -- Perform more rigorous check for distance-too-far in inffast.c -- Add Z_BLOCK flush option to return from inflate at block boundary -- Set strm->data_type on return from inflate - - Indicate bits unused, if at block boundary, and if in last block -- Replace size_t with ptrdiff_t in crc32.c, and check for correct size -- Add condition so old NO_DEFLATE define still works for compatibility -- FAQ update regarding the Windows DLL [Truta] -- INDEX update: add qnx entry, remove aix entry [Truta] -- Install zlib.3 into mandir [Wilson] -- Move contrib/zlib_dll_FAQ.txt to win32/DLL_FAQ.txt; update [Truta] -- Adapt the zlib interface to the new DLL convention guidelines [Truta] -- Introduce ZLIB_WINAPI macro to allow the export of functions using - the WINAPI calling convention, for Visual Basic [Vollant, Truta] -- Update msdos and win32 scripts and makefiles [Truta] -- Export symbols by name, not by ordinal, in win32/zlib.def [Truta] -- Add contrib/ada [Anisimkov] -- Move asm files from contrib/vstudio/vc70_32 to contrib/asm386 [Truta] -- Rename contrib/asm386 to contrib/masmx86 [Truta, Vollant] -- Add contrib/masm686 [Truta] -- Fix offsets in contrib/inflate86 and contrib/masmx86/inffas32.asm - [Truta, Vollant] -- Update contrib/delphi; rename to contrib/pascal; add example [Truta] -- Remove contrib/delphi2; add a new contrib/delphi [Truta] -- Avoid inclusion of the nonstandard in contrib/iostream, - and fix some method prototypes [Truta] -- Fix the ZCR_SEED2 constant to avoid warnings in contrib/minizip - [Truta] -- Avoid the use of backslash (\) in contrib/minizip [Vollant] -- Fix file time handling in contrib/untgz; update makefiles [Truta] -- Update contrib/vstudio/vc70_32 to comply with the new DLL guidelines - [Vollant] -- Remove contrib/vstudio/vc15_16 [Vollant] -- Rename contrib/vstudio/vc70_32 to contrib/vstudio/vc7 [Truta] -- Update README.contrib [Truta] -- Invert the assignment order of match_head and s->prev[...] in - INSERT_STRING [Truta] -- Compare TOO_FAR with 32767 instead of 32768, to avoid 16-bit warnings - [Truta] -- Compare function pointers with 0, not with NULL or Z_NULL [Truta] -- Fix prototype of syncsearch in inflate.c [Truta] -- Introduce ASMINF macro to be enabled when using an ASM implementation - of inflate_fast [Truta] -- Change NO_DEFLATE to NO_GZCOMPRESS [Truta] -- Modify test_gzio in example.c to take a single file name as a - parameter [Truta] -- Exit the example.c program if gzopen fails [Truta] -- Add type casts around strlen in example.c [Truta] -- Remove casting to sizeof in minigzip.c; give a proper type - to the variable compared with SUFFIX_LEN [Truta] -- Update definitions of STDC and STDC99 in zconf.h [Truta] -- Synchronize zconf.h with the new Windows DLL interface [Truta] -- Use SYS16BIT instead of __32BIT__ to distinguish between - 16- and 32-bit platforms [Truta] -- Use far memory allocators in small 16-bit memory models for - Turbo C [Truta] -- Add info about the use of ASMV, ASMINF and ZLIB_WINAPI in - zlibCompileFlags [Truta] -- Cygwin has vsnprintf [Wilson] -- In Windows16, OS_CODE is 0, as in MSDOS [Truta] -- In Cygwin, OS_CODE is 3 (Unix), not 11 (Windows32) [Wilson] - -Changes in 1.2.0.4 (10 August 2003) -- Minor FAQ updates -- Be more strict when checking inflateInit2's windowBits parameter -- Change NO_GUNZIP compile option to NO_GZIP to cover deflate as well -- Add gzip wrapper option to deflateInit2 using windowBits -- Add updated QNX rule in configure and qnx directory [Bonnefoy] -- Make inflate distance-too-far checks more rigorous -- Clean up FAR usage in inflate -- Add casting to sizeof() in gzio.c and minigzip.c - -Changes in 1.2.0.3 (19 July 2003) -- Fix silly error in gzungetc() implementation [Vollant] -- Update contrib/minizip and contrib/vstudio [Vollant] -- Fix printf format in example.c -- Correct cdecl support in zconf.in.h [Anisimkov] -- Minor FAQ updates - -Changes in 1.2.0.2 (13 July 2003) -- Add ZLIB_VERNUM in zlib.h for numerical preprocessor comparisons -- Attempt to avoid warnings in crc32.c for pointer-int conversion -- Add AIX to configure, remove aix directory [Bakker] -- Add some casts to minigzip.c -- Improve checking after insecure sprintf() or vsprintf() calls -- Remove #elif's from crc32.c -- Change leave label to inf_leave in inflate.c and infback.c to avoid - library conflicts -- Remove inflate gzip decoding by default--only enable gzip decoding by - special request for stricter backward compatibility -- Add zlibCompileFlags() function to return compilation information -- More typecasting in deflate.c to avoid warnings -- Remove leading underscore from _Capital #defines [Truta] -- Fix configure to link shared library when testing -- Add some Windows CE target adjustments [Mai] -- Remove #define ZLIB_DLL in zconf.h [Vollant] -- Add zlib.3 [Rodgers] -- Update RFC URL in deflate.c and algorithm.txt [Mai] -- Add zlib_dll_FAQ.txt to contrib [Truta] -- Add UL to some constants [Truta] -- Update minizip and vstudio [Vollant] -- Remove vestigial NEED_DUMMY_RETURN from zconf.in.h -- Expand use of NO_DUMMY_DECL to avoid all dummy structures -- Added iostream3 to contrib [Schwardt] -- Replace rewind() with fseek() for WinCE [Truta] -- Improve setting of zlib format compression level flags - - Report 0 for huffman and rle strategies and for level == 0 or 1 - - Report 2 only for level == 6 -- Only deal with 64K limit when necessary at compile time [Truta] -- Allow TOO_FAR check to be turned off at compile time [Truta] -- Add gzclearerr() function [Souza] -- Add gzungetc() function - -Changes in 1.2.0.1 (17 March 2003) -- Add Z_RLE strategy for run-length encoding [Truta] - - When Z_RLE requested, restrict matches to distance one - - Update zlib.h, minigzip.c, gzopen(), gzdopen() for Z_RLE -- Correct FASTEST compilation to allow level == 0 -- Clean up what gets compiled for FASTEST -- Incorporate changes to zconf.in.h [Vollant] - - Refine detection of Turbo C need for dummy returns - - Refine ZLIB_DLL compilation - - Include additional header file on VMS for off_t typedef -- Try to use _vsnprintf where it supplants vsprintf [Vollant] -- Add some casts in inffast.c -- Enchance comments in zlib.h on what happens if gzprintf() tries to - write more than 4095 bytes before compression -- Remove unused state from inflateBackEnd() -- Remove exit(0) from minigzip.c, example.c -- Get rid of all those darn tabs -- Add "check" target to Makefile.in that does the same thing as "test" -- Add "mostlyclean" and "maintainer-clean" targets to Makefile.in -- Update contrib/inflate86 [Anderson] -- Update contrib/testzlib, contrib/vstudio, contrib/minizip [Vollant] -- Add msdos and win32 directories with makefiles [Truta] -- More additions and improvements to the FAQ - -Changes in 1.2.0 (9 March 2003) -- New and improved inflate code - - About 20% faster - - Does not allocate 32K window unless and until needed - - Automatically detects and decompresses gzip streams - - Raw inflate no longer needs an extra dummy byte at end - - Added inflateBack functions using a callback interface--even faster - than inflate, useful for file utilities (gzip, zip) - - Added inflateCopy() function to record state for random access on - externally generated deflate streams (e.g. in gzip files) - - More readable code (I hope) -- New and improved crc32() - - About 50% faster, thanks to suggestions from Rodney Brown -- Add deflateBound() and compressBound() functions -- Fix memory leak in deflateInit2() -- Permit setting dictionary for raw deflate (for parallel deflate) -- Fix const declaration for gzwrite() -- Check for some malloc() failures in gzio.c -- Fix bug in gzopen() on single-byte file 0x1f -- Fix bug in gzread() on concatenated file with 0x1f at end of buffer - and next buffer doesn't start with 0x8b -- Fix uncompress() to return Z_DATA_ERROR on truncated input -- Free memory at end of example.c -- Remove MAX #define in trees.c (conflicted with some libraries) -- Fix static const's in deflate.c, gzio.c, and zutil.[ch] -- Declare malloc() and free() in gzio.c if STDC not defined -- Use malloc() instead of calloc() in zutil.c if int big enough -- Define STDC for AIX -- Add aix/ with approach for compiling shared library on AIX -- Add HP-UX support for shared libraries in configure -- Add OpenUNIX support for shared libraries in configure -- Use $cc instead of gcc to build shared library -- Make prefix directory if needed when installing -- Correct Macintosh avoidance of typedef Byte in zconf.h -- Correct Turbo C memory allocation when under Linux -- Use libz.a instead of -lz in Makefile (assure use of compiled library) -- Update configure to check for snprintf or vsnprintf functions and their - return value, warn during make if using an insecure function -- Fix configure problem with compile-time knowledge of HAVE_UNISTD_H that - is lost when library is used--resolution is to build new zconf.h -- Documentation improvements (in zlib.h): - - Document raw deflate and inflate - - Update RFCs URL - - Point out that zlib and gzip formats are different - - Note that Z_BUF_ERROR is not fatal - - Document string limit for gzprintf() and possible buffer overflow - - Note requirement on avail_out when flushing - - Note permitted values of flush parameter of inflate() -- Add some FAQs (and even answers) to the FAQ -- Add contrib/inflate86/ for x86 faster inflate -- Add contrib/blast/ for PKWare Data Compression Library decompression -- Add contrib/puff/ simple inflate for deflate format description - -Changes in 1.1.4 (11 March 2002) -- ZFREE was repeated on same allocation on some error conditions. - This creates a security problem described in - http://www.zlib.org/advisory-2002-03-11.txt -- Returned incorrect error (Z_MEM_ERROR) on some invalid data -- Avoid accesses before window for invalid distances with inflate window - less than 32K. -- force windowBits > 8 to avoid a bug in the encoder for a window size - of 256 bytes. (A complete fix will be available in 1.1.5). - -Changes in 1.1.3 (9 July 1998) -- fix "an inflate input buffer bug that shows up on rare but persistent - occasions" (Mark) -- fix gzread and gztell for concatenated .gz files (Didier Le Botlan) -- fix gzseek(..., SEEK_SET) in write mode -- fix crc check after a gzeek (Frank Faubert) -- fix miniunzip when the last entry in a zip file is itself a zip file - (J Lillge) -- add contrib/asm586 and contrib/asm686 (Brian Raiter) - See http://www.muppetlabs.com/~breadbox/software/assembly.html -- add support for Delphi 3 in contrib/delphi (Bob Dellaca) -- add support for C++Builder 3 and Delphi 3 in contrib/delphi2 (Davide Moretti) -- do not exit prematurely in untgz if 0 at start of block (Magnus Holmgren) -- use macro EXTERN instead of extern to support DLL for BeOS (Sander Stoks) -- added a FAQ file - -- Support gzdopen on Mac with Metrowerks (Jason Linhart) -- Do not redefine Byte on Mac (Brad Pettit & Jason Linhart) -- define SEEK_END too if SEEK_SET is not defined (Albert Chin-A-Young) -- avoid some warnings with Borland C (Tom Tanner) -- fix a problem in contrib/minizip/zip.c for 16-bit MSDOS (Gilles Vollant) -- emulate utime() for WIN32 in contrib/untgz (Gilles Vollant) -- allow several arguments to configure (Tim Mooney, Frodo Looijaard) -- use libdir and includedir in Makefile.in (Tim Mooney) -- support shared libraries on OSF1 V4 (Tim Mooney) -- remove so_locations in "make clean" (Tim Mooney) -- fix maketree.c compilation error (Glenn, Mark) -- Python interface to zlib now in Python 1.5 (Jeremy Hylton) -- new Makefile.riscos (Rich Walker) -- initialize static descriptors in trees.c for embedded targets (Nick Smith) -- use "foo-gz" in example.c for RISCOS and VMS (Nick Smith) -- add the OS/2 files in Makefile.in too (Andrew Zabolotny) -- fix fdopen and halloc macros for Microsoft C 6.0 (Tom Lane) -- fix maketree.c to allow clean compilation of inffixed.h (Mark) -- fix parameter check in deflateCopy (Gunther Nikl) -- cleanup trees.c, use compressed_len only in debug mode (Christian Spieler) -- Many portability patches by Christian Spieler: - . zutil.c, zutil.h: added "const" for zmem* - . Make_vms.com: fixed some typos - . Make_vms.com: msdos/Makefile.*: removed zutil.h from some dependency lists - . msdos/Makefile.msc: remove "default rtl link library" info from obj files - . msdos/Makefile.*: use model-dependent name for the built zlib library - . msdos/Makefile.emx, nt/Makefile.emx, nt/Makefile.gcc: - new makefiles, for emx (DOS/OS2), emx&rsxnt and mingw32 (Windows 9x / NT) -- use define instead of typedef for Bytef also for MSC small/medium (Tom Lane) -- replace __far with _far for better portability (Christian Spieler, Tom Lane) -- fix test for errno.h in configure (Tim Newsham) - -Changes in 1.1.2 (19 March 98) -- added contrib/minzip, mini zip and unzip based on zlib (Gilles Vollant) - See http://www.winimage.com/zLibDll/unzip.html -- preinitialize the inflate tables for fixed codes, to make the code - completely thread safe (Mark) -- some simplifications and slight speed-up to the inflate code (Mark) -- fix gzeof on non-compressed files (Allan Schrum) -- add -std1 option in configure for OSF1 to fix gzprintf (Martin Mokrejs) -- use default value of 4K for Z_BUFSIZE for 16-bit MSDOS (Tim Wegner + Glenn) -- added os2/Makefile.def and os2/zlib.def (Andrew Zabolotny) -- add shared lib support for UNIX_SV4.2MP (MATSUURA Takanori) -- do not wrap extern "C" around system includes (Tom Lane) -- mention zlib binding for TCL in README (Andreas Kupries) -- added amiga/Makefile.pup for Amiga powerUP SAS/C PPC (Andreas Kleinert) -- allow "make install prefix=..." even after configure (Glenn Randers-Pehrson) -- allow "configure --prefix $HOME" (Tim Mooney) -- remove warnings in example.c and gzio.c (Glenn Randers-Pehrson) -- move Makefile.sas to amiga/Makefile.sas - -Changes in 1.1.1 (27 Feb 98) -- fix macros _tr_tally_* in deflate.h for debug mode (Glenn Randers-Pehrson) -- remove block truncation heuristic which had very marginal effect for zlib - (smaller lit_bufsize than in gzip 1.2.4) and degraded a little the - compression ratio on some files. This also allows inlining _tr_tally for - matches in deflate_slow. -- added msdos/Makefile.w32 for WIN32 Microsoft Visual C++ (Bob Frazier) - -Changes in 1.1.0 (24 Feb 98) -- do not return STREAM_END prematurely in inflate (John Bowler) -- revert to the zlib 1.0.8 inflate to avoid the gcc 2.8.0 bug (Jeremy Buhler) -- compile with -DFASTEST to get compression code optimized for speed only -- in minigzip, try mmap'ing the input file first (Miguel Albrecht) -- increase size of I/O buffers in minigzip.c and gzio.c (not a big gain - on Sun but significant on HP) - -- add a pointer to experimental unzip library in README (Gilles Vollant) -- initialize variable gcc in configure (Chris Herborth) - -Changes in 1.0.9 (17 Feb 1998) -- added gzputs and gzgets functions -- do not clear eof flag in gzseek (Mark Diekhans) -- fix gzseek for files in transparent mode (Mark Diekhans) -- do not assume that vsprintf returns the number of bytes written (Jens Krinke) -- replace EXPORT with ZEXPORT to avoid conflict with other programs -- added compress2 in zconf.h, zlib.def, zlib.dnt -- new asm code from Gilles Vollant in contrib/asm386 -- simplify the inflate code (Mark): - . Replace ZALLOC's in huft_build() with single ZALLOC in inflate_blocks_new() - . ZALLOC the length list in inflate_trees_fixed() instead of using stack - . ZALLOC the value area for huft_build() instead of using stack - . Simplify Z_FINISH check in inflate() - -- Avoid gcc 2.8.0 comparison bug a little differently than zlib 1.0.8 -- in inftrees.c, avoid cc -O bug on HP (Farshid Elahi) -- in zconf.h move the ZLIB_DLL stuff earlier to avoid problems with - the declaration of FAR (Gilles VOllant) -- install libz.so* with mode 755 (executable) instead of 644 (Marc Lehmann) -- read_buf buf parameter of type Bytef* instead of charf* -- zmemcpy parameters are of type Bytef*, not charf* (Joseph Strout) -- do not redeclare unlink in minigzip.c for WIN32 (John Bowler) -- fix check for presence of directories in "make install" (Ian Willis) - -Changes in 1.0.8 (27 Jan 1998) -- fixed offsets in contrib/asm386/gvmat32.asm (Gilles Vollant) -- fix gzgetc and gzputc for big endian systems (Markus Oberhumer) -- added compress2() to allow setting the compression level -- include sys/types.h to get off_t on some systems (Marc Lehmann & QingLong) -- use constant arrays for the static trees in trees.c instead of computing - them at run time (thanks to Ken Raeburn for this suggestion). To create - trees.h, compile with GEN_TREES_H and run "make test". -- check return code of example in "make test" and display result -- pass minigzip command line options to file_compress -- simplifying code of inflateSync to avoid gcc 2.8 bug - -- support CC="gcc -Wall" in configure -s (QingLong) -- avoid a flush caused by ftell in gzopen for write mode (Ken Raeburn) -- fix test for shared library support to avoid compiler warnings -- zlib.lib -> zlib.dll in msdos/zlib.rc (Gilles Vollant) -- check for TARGET_OS_MAC in addition to MACOS (Brad Pettit) -- do not use fdopen for Metrowerks on Mac (Brad Pettit)) -- add checks for gzputc and gzputc in example.c -- avoid warnings in gzio.c and deflate.c (Andreas Kleinert) -- use const for the CRC table (Ken Raeburn) -- fixed "make uninstall" for shared libraries -- use Tracev instead of Trace in infblock.c -- in example.c use correct compressed length for test_sync -- suppress +vnocompatwarnings in configure for HPUX (not always supported) - -Changes in 1.0.7 (20 Jan 1998) -- fix gzseek which was broken in write mode -- return error for gzseek to negative absolute position -- fix configure for Linux (Chun-Chung Chen) -- increase stack space for MSC (Tim Wegner) -- get_crc_table and inflateSyncPoint are EXPORTed (Gilles Vollant) -- define EXPORTVA for gzprintf (Gilles Vollant) -- added man page zlib.3 (Rick Rodgers) -- for contrib/untgz, fix makedir() and improve Makefile - -- check gzseek in write mode in example.c -- allocate extra buffer for seeks only if gzseek is actually called -- avoid signed/unsigned comparisons (Tim Wegner, Gilles Vollant) -- add inflateSyncPoint in zconf.h -- fix list of exported functions in nt/zlib.dnt and mdsos/zlib.def - -Changes in 1.0.6 (19 Jan 1998) -- add functions gzprintf, gzputc, gzgetc, gztell, gzeof, gzseek, gzrewind and - gzsetparams (thanks to Roland Giersig and Kevin Ruland for some of this code) -- Fix a deflate bug occurring only with compression level 0 (thanks to - Andy Buckler for finding this one). -- In minigzip, pass transparently also the first byte for .Z files. -- return Z_BUF_ERROR instead of Z_OK if output buffer full in uncompress() -- check Z_FINISH in inflate (thanks to Marc Schluper) -- Implement deflateCopy (thanks to Adam Costello) -- make static libraries by default in configure, add --shared option. -- move MSDOS or Windows specific files to directory msdos -- suppress the notion of partial flush to simplify the interface - (but the symbol Z_PARTIAL_FLUSH is kept for compatibility with 1.0.4) -- suppress history buffer provided by application to simplify the interface - (this feature was not implemented anyway in 1.0.4) -- next_in and avail_in must be initialized before calling inflateInit or - inflateInit2 -- add EXPORT in all exported functions (for Windows DLL) -- added Makefile.nt (thanks to Stephen Williams) -- added the unsupported "contrib" directory: - contrib/asm386/ by Gilles Vollant - 386 asm code replacing longest_match(). - contrib/iostream/ by Kevin Ruland - A C++ I/O streams interface to the zlib gz* functions - contrib/iostream2/ by Tyge Løvset - Another C++ I/O streams interface - contrib/untgz/ by "Pedro A. Aranda Guti\irrez" - A very simple tar.gz file extractor using zlib - contrib/visual-basic.txt by Carlos Rios - How to use compress(), uncompress() and the gz* functions from VB. -- pass params -f (filtered data), -h (huffman only), -1 to -9 (compression - level) in minigzip (thanks to Tom Lane) - -- use const for rommable constants in deflate -- added test for gzseek and gztell in example.c -- add undocumented function inflateSyncPoint() (hack for Paul Mackerras) -- add undocumented function zError to convert error code to string - (for Tim Smithers) -- Allow compilation of gzio with -DNO_DEFLATE to avoid the compression code. -- Use default memcpy for Symantec MSDOS compiler. -- Add EXPORT keyword for check_func (needed for Windows DLL) -- add current directory to LD_LIBRARY_PATH for "make test" -- create also a link for libz.so.1 -- added support for FUJITSU UXP/DS (thanks to Toshiaki Nomura) -- use $(SHAREDLIB) instead of libz.so in Makefile.in (for HPUX) -- added -soname for Linux in configure (Chun-Chung Chen, -- assign numbers to the exported functions in zlib.def (for Windows DLL) -- add advice in zlib.h for best usage of deflateSetDictionary -- work around compiler bug on Atari (cast Z_NULL in call of s->checkfn) -- allow compilation with ANSI keywords only enabled for TurboC in large model -- avoid "versionString"[0] (Borland bug) -- add NEED_DUMMY_RETURN for Borland -- use variable z_verbose for tracing in debug mode (L. Peter Deutsch). -- allow compilation with CC -- defined STDC for OS/2 (David Charlap) -- limit external names to 8 chars for MVS (Thomas Lund) -- in minigzip.c, use static buffers only for 16-bit systems -- fix suffix check for "minigzip -d foo.gz" -- do not return an error for the 2nd of two consecutive gzflush() (Felix Lee) -- use _fdopen instead of fdopen for MSC >= 6.0 (Thomas Fanslau) -- added makelcc.bat for lcc-win32 (Tom St Denis) -- in Makefile.dj2, use copy and del instead of install and rm (Frank Donahoe) -- Avoid expanded $Id$. Use "rcs -kb" or "cvs admin -kb" to avoid Id expansion. -- check for unistd.h in configure (for off_t) -- remove useless check parameter in inflate_blocks_free -- avoid useless assignment of s->check to itself in inflate_blocks_new -- do not flush twice in gzclose (thanks to Ken Raeburn) -- rename FOPEN as F_OPEN to avoid clash with /usr/include/sys/file.h -- use NO_ERRNO_H instead of enumeration of operating systems with errno.h -- work around buggy fclose on pipes for HP/UX -- support zlib DLL with BORLAND C++ 5.0 (thanks to Glenn Randers-Pehrson) -- fix configure if CC is already equal to gcc - -Changes in 1.0.5 (3 Jan 98) -- Fix inflate to terminate gracefully when fed corrupted or invalid data -- Use const for rommable constants in inflate -- Eliminate memory leaks on error conditions in inflate -- Removed some vestigial code in inflate -- Update web address in README - -Changes in 1.0.4 (24 Jul 96) -- In very rare conditions, deflate(s, Z_FINISH) could fail to produce an EOF - bit, so the decompressor could decompress all the correct data but went - on to attempt decompressing extra garbage data. This affected minigzip too. -- zlibVersion and gzerror return const char* (needed for DLL) -- port to RISCOS (no fdopen, no multiple dots, no unlink, no fileno) -- use z_error only for DEBUG (avoid problem with DLLs) - -Changes in 1.0.3 (2 Jul 96) -- use z_streamp instead of z_stream *, which is now a far pointer in MSDOS - small and medium models; this makes the library incompatible with previous - versions for these models. (No effect in large model or on other systems.) -- return OK instead of BUF_ERROR if previous deflate call returned with - avail_out as zero but there is nothing to do -- added memcmp for non STDC compilers -- define NO_DUMMY_DECL for more Mac compilers (.h files merged incorrectly) -- define __32BIT__ if __386__ or i386 is defined (pb. with Watcom and SCO) -- better check for 16-bit mode MSC (avoids problem with Symantec) - -Changes in 1.0.2 (23 May 96) -- added Windows DLL support -- added a function zlibVersion (for the DLL support) -- fixed declarations using Bytef in infutil.c (pb with MSDOS medium model) -- Bytef is define's instead of typedef'd only for Borland C -- avoid reading uninitialized memory in example.c -- mention in README that the zlib format is now RFC1950 -- updated Makefile.dj2 -- added algorithm.doc - -Changes in 1.0.1 (20 May 96) [1.0 skipped to avoid confusion] -- fix array overlay in deflate.c which sometimes caused bad compressed data -- fix inflate bug with empty stored block -- fix MSDOS medium model which was broken in 0.99 -- fix deflateParams() which could generate bad compressed data. -- Bytef is define'd instead of typedef'ed (work around Borland bug) -- added an INDEX file -- new makefiles for DJGPP (Makefile.dj2), 32-bit Borland (Makefile.b32), - Watcom (Makefile.wat), Amiga SAS/C (Makefile.sas) -- speed up adler32 for modern machines without auto-increment -- added -ansi for IRIX in configure -- static_init_done in trees.c is an int -- define unlink as delete for VMS -- fix configure for QNX -- add configure branch for SCO and HPUX -- avoid many warnings (unused variables, dead assignments, etc...) -- no fdopen for BeOS -- fix the Watcom fix for 32 bit mode (define FAR as empty) -- removed redefinition of Byte for MKWERKS -- work around an MWKERKS bug (incorrect merge of all .h files) - -Changes in 0.99 (27 Jan 96) -- allow preset dictionary shared between compressor and decompressor -- allow compression level 0 (no compression) -- add deflateParams in zlib.h: allow dynamic change of compression level - and compression strategy. -- test large buffers and deflateParams in example.c -- add optional "configure" to build zlib as a shared library -- suppress Makefile.qnx, use configure instead -- fixed deflate for 64-bit systems (detected on Cray) -- fixed inflate_blocks for 64-bit systems (detected on Alpha) -- declare Z_DEFLATED in zlib.h (possible parameter for deflateInit2) -- always return Z_BUF_ERROR when deflate() has nothing to do -- deflateInit and inflateInit are now macros to allow version checking -- prefix all global functions and types with z_ with -DZ_PREFIX -- make falloc completely reentrant (inftrees.c) -- fixed very unlikely race condition in ct_static_init -- free in reverse order of allocation to help memory manager -- use zlib-1.0/* instead of zlib/* inside the tar.gz -- make zlib warning-free with "gcc -O3 -Wall -Wwrite-strings -Wpointer-arith - -Wconversion -Wstrict-prototypes -Wmissing-prototypes" -- allow gzread on concatenated .gz files -- deflateEnd now returns Z_DATA_ERROR if it was premature -- deflate is finally (?) fully deterministic (no matches beyond end of input) -- Document Z_SYNC_FLUSH -- add uninstall in Makefile -- Check for __cpluplus in zlib.h -- Better test in ct_align for partial flush -- avoid harmless warnings for Borland C++ -- initialize hash_head in deflate.c -- avoid warning on fdopen (gzio.c) for HP cc -Aa -- include stdlib.h for STDC compilers -- include errno.h for Cray -- ignore error if ranlib doesn't exist -- call ranlib twice for NeXTSTEP -- use exec_prefix instead of prefix for libz.a -- renamed ct_* as _tr_* to avoid conflict with applications -- clear z->msg in inflateInit2 before any error return -- initialize opaque in example.c, gzio.c, deflate.c and inflate.c -- fixed typo in zconf.h (_GNUC__ => __GNUC__) -- check for WIN32 in zconf.h and zutil.c (avoid farmalloc in 32-bit mode) -- fix typo in Make_vms.com (f$trnlnm -> f$getsyi) -- in fcalloc, normalize pointer if size > 65520 bytes -- don't use special fcalloc for 32 bit Borland C++ -- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc... -- use Z_BINARY instead of BINARY -- document that gzclose after gzdopen will close the file -- allow "a" as mode in gzopen. -- fix error checking in gzread -- allow skipping .gz extra-field on pipes -- added reference to Perl interface in README -- put the crc table in FAR data (I dislike more and more the medium model :) -- added get_crc_table -- added a dimension to all arrays (Borland C can't count). -- workaround Borland C bug in declaration of inflate_codes_new & inflate_fast -- guard against multiple inclusion of *.h (for precompiled header on Mac) -- Watcom C pretends to be Microsoft C small model even in 32 bit mode. -- don't use unsized arrays to avoid silly warnings by Visual C++: - warning C4746: 'inflate_mask' : unsized array treated as '__far' - (what's wrong with far data in far model?). -- define enum out of inflate_blocks_state to allow compilation with C++ - -Changes in 0.95 (16 Aug 95) -- fix MSDOS small and medium model (now easier to adapt to any compiler) -- inlined send_bits -- fix the final (:-) bug for deflate with flush (output was correct but - not completely flushed in rare occasions). -- default window size is same for compression and decompression - (it's now sufficient to set MAX_WBITS in zconf.h). -- voidp -> voidpf and voidnp -> voidp (for consistency with other - typedefs and because voidnp was not near in large model). - -Changes in 0.94 (13 Aug 95) -- support MSDOS medium model -- fix deflate with flush (could sometimes generate bad output) -- fix deflateReset (zlib header was incorrectly suppressed) -- added support for VMS -- allow a compression level in gzopen() -- gzflush now calls fflush -- For deflate with flush, flush even if no more input is provided. -- rename libgz.a as libz.a -- avoid complex expression in infcodes.c triggering Turbo C bug -- work around a problem with gcc on Alpha (in INSERT_STRING) -- don't use inline functions (problem with some gcc versions) -- allow renaming of Byte, uInt, etc... with #define. -- avoid warning about (unused) pointer before start of array in deflate.c -- avoid various warnings in gzio.c, example.c, infblock.c, adler32.c, zutil.c -- avoid reserved word 'new' in trees.c - -Changes in 0.93 (25 June 95) -- temporarily disable inline functions -- make deflate deterministic -- give enough lookahead for PARTIAL_FLUSH -- Set binary mode for stdin/stdout in minigzip.c for OS/2 -- don't even use signed char in inflate (not portable enough) -- fix inflate memory leak for segmented architectures - -Changes in 0.92 (3 May 95) -- don't assume that char is signed (problem on SGI) -- Clear bit buffer when starting a stored block -- no memcpy on Pyramid -- suppressed inftest.c -- optimized fill_window, put longest_match inline for gcc -- optimized inflate on stored blocks. -- untabify all sources to simplify patches - -Changes in 0.91 (2 May 95) -- Default MEM_LEVEL is 8 (not 9 for Unix) as documented in zlib.h -- Document the memory requirements in zconf.h -- added "make install" -- fix sync search logic in inflateSync -- deflate(Z_FULL_FLUSH) now works even if output buffer too short -- after inflateSync, don't scare people with just "lo world" -- added support for DJGPP - -Changes in 0.9 (1 May 95) -- don't assume that zalloc clears the allocated memory (the TurboC bug - was Mark's bug after all :) -- let again gzread copy uncompressed data unchanged (was working in 0.71) -- deflate(Z_FULL_FLUSH), inflateReset and inflateSync are now fully implemented -- added a test of inflateSync in example.c -- moved MAX_WBITS to zconf.h because users might want to change that. -- document explicitly that zalloc(64K) on MSDOS must return a normalized - pointer (zero offset) -- added Makefiles for Microsoft C, Turbo C, Borland C++ -- faster crc32() - -Changes in 0.8 (29 April 95) -- added fast inflate (inffast.c) -- deflate(Z_FINISH) now returns Z_STREAM_END when done. Warning: this - is incompatible with previous versions of zlib which returned Z_OK. -- work around a TurboC compiler bug (bad code for b << 0, see infutil.h) - (actually that was not a compiler bug, see 0.81 above) -- gzread no longer reads one extra byte in certain cases -- In gzio destroy(), don't reference a freed structure -- avoid many warnings for MSDOS -- avoid the ERROR symbol which is used by MS Windows - -Changes in 0.71 (14 April 95) -- Fixed more MSDOS compilation problems :( There is still a bug with - TurboC large model. - -Changes in 0.7 (14 April 95) -- Added full inflate support. -- Simplified the crc32() interface. The pre- and post-conditioning - (one's complement) is now done inside crc32(). WARNING: this is - incompatible with previous versions; see zlib.h for the new usage. - -Changes in 0.61 (12 April 95) -- workaround for a bug in TurboC. example and minigzip now work on MSDOS. - -Changes in 0.6 (11 April 95) -- added minigzip.c -- added gzdopen to reopen a file descriptor as gzFile -- added transparent reading of non-gziped files in gzread. -- fixed bug in gzread (don't read crc as data) -- fixed bug in destroy (gzio.c) (don't return Z_STREAM_END for gzclose). -- don't allocate big arrays in the stack (for MSDOS) -- fix some MSDOS compilation problems - -Changes in 0.5: -- do real compression in deflate.c. Z_PARTIAL_FLUSH is supported but - not yet Z_FULL_FLUSH. -- support decompression but only in a single step (forced Z_FINISH) -- added opaque object for zalloc and zfree. -- added deflateReset and inflateReset -- added a variable zlib_version for consistency checking. -- renamed the 'filter' parameter of deflateInit2 as 'strategy'. - Added Z_FILTERED and Z_HUFFMAN_ONLY constants. - -Changes in 0.4: -- avoid "zip" everywhere, use zlib instead of ziplib. -- suppress Z_BLOCK_FLUSH, interpret Z_PARTIAL_FLUSH as block flush - if compression method == 8. -- added adler32 and crc32 -- renamed deflateOptions as deflateInit2, call one or the other but not both -- added the method parameter for deflateInit2. -- added inflateInit2 -- simplied considerably deflateInit and inflateInit by not supporting - user-provided history buffer. This is supported only in deflateInit2 - and inflateInit2. - -Changes in 0.3: -- prefix all macro names with Z_ -- use Z_FINISH instead of deflateEnd to finish compression. -- added Z_HUFFMAN_ONLY -- added gzerror() diff --git a/contrib/zlib/FAQ b/contrib/zlib/FAQ deleted file mode 100644 index 99b7cf92e..000000000 --- a/contrib/zlib/FAQ +++ /dev/null @@ -1,368 +0,0 @@ - - Frequently Asked Questions about zlib - - -If your question is not there, please check the zlib home page -http://zlib.net/ which may have more recent information. -The lastest zlib FAQ is at http://zlib.net/zlib_faq.html - - - 1. Is zlib Y2K-compliant? - - Yes. zlib doesn't handle dates. - - 2. Where can I get a Windows DLL version? - - The zlib sources can be compiled without change to produce a DLL. See the - file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the - precompiled DLL are found in the zlib web site at http://zlib.net/ . - - 3. Where can I get a Visual Basic interface to zlib? - - See - * http://marknelson.us/1997/01/01/zlib-engine/ - * win32/DLL_FAQ.txt in the zlib distribution - - 4. compress() returns Z_BUF_ERROR. - - Make sure that before the call of compress(), the length of the compressed - buffer is equal to the available size of the compressed buffer and not - zero. For Visual Basic, check that this parameter is passed by reference - ("as any"), not by value ("as long"). - - 5. deflate() or inflate() returns Z_BUF_ERROR. - - Before making the call, make sure that avail_in and avail_out are not zero. - When setting the parameter flush equal to Z_FINISH, also make sure that - avail_out is big enough to allow processing all pending input. Note that a - Z_BUF_ERROR is not fatal--another call to deflate() or inflate() can be - made with more input or output space. A Z_BUF_ERROR may in fact be - unavoidable depending on how the functions are used, since it is not - possible to tell whether or not there is more output pending when - strm.avail_out returns with zero. See http://zlib.net/zlib_how.html for a - heavily annotated example. - - 6. Where's the zlib documentation (man pages, etc.)? - - It's in zlib.h . Examples of zlib usage are in the files test/example.c - and test/minigzip.c, with more in examples/ . - - 7. Why don't you use GNU autoconf or libtool or ...? - - Because we would like to keep zlib as a very small and simple package. - zlib is rather portable and doesn't need much configuration. - - 8. I found a bug in zlib. - - Most of the time, such problems are due to an incorrect usage of zlib. - Please try to reproduce the problem with a small program and send the - corresponding source to us at zlib@gzip.org . Do not send multi-megabyte - data files without prior agreement. - - 9. Why do I get "undefined reference to gzputc"? - - If "make test" produces something like - - example.o(.text+0x154): undefined reference to `gzputc' - - check that you don't have old files libz.* in /usr/lib, /usr/local/lib or - /usr/X11R6/lib. Remove any old versions, then do "make install". - -10. I need a Delphi interface to zlib. - - See the contrib/delphi directory in the zlib distribution. - -11. Can zlib handle .zip archives? - - Not by itself, no. See the directory contrib/minizip in the zlib - distribution. - -12. Can zlib handle .Z files? - - No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt - the code of uncompress on your own. - -13. How can I make a Unix shared library? - - By default a shared (and a static) library is built for Unix. So: - - make distclean - ./configure - make - -14. How do I install a shared zlib library on Unix? - - After the above, then: - - make install - - However, many flavors of Unix come with a shared zlib already installed. - Before going to the trouble of compiling a shared version of zlib and - trying to install it, you may want to check if it's already there! If you - can #include , it's there. The -lz option will probably link to - it. You can check the version at the top of zlib.h or with the - ZLIB_VERSION symbol defined in zlib.h . - -15. I have a question about OttoPDF. - - We are not the authors of OttoPDF. The real author is on the OttoPDF web - site: Joel Hainley, jhainley@myndkryme.com. - -16. Can zlib decode Flate data in an Adobe PDF file? - - Yes. See http://www.pdflib.com/ . To modify PDF forms, see - http://sourceforge.net/projects/acroformtool/ . - -17. Why am I getting this "register_frame_info not found" error on Solaris? - - After installing zlib 1.1.4 on Solaris 2.6, running applications using zlib - generates an error such as: - - ld.so.1: rpm: fatal: relocation error: file /usr/local/lib/libz.so: - symbol __register_frame_info: referenced symbol not found - - The symbol __register_frame_info is not part of zlib, it is generated by - the C compiler (cc or gcc). You must recompile applications using zlib - which have this problem. This problem is specific to Solaris. See - http://www.sunfreeware.com for Solaris versions of zlib and applications - using zlib. - -18. Why does gzip give an error on a file I make with compress/deflate? - - The compress and deflate functions produce data in the zlib format, which - is different and incompatible with the gzip format. The gz* functions in - zlib on the other hand use the gzip format. Both the zlib and gzip formats - use the same compressed data format internally, but have different headers - and trailers around the compressed data. - -19. Ok, so why are there two different formats? - - The gzip format was designed to retain the directory information about a - single file, such as the name and last modification date. The zlib format - on the other hand was designed for in-memory and communication channel - applications, and has a much more compact header and trailer and uses a - faster integrity check than gzip. - -20. Well that's nice, but how do I make a gzip file in memory? - - You can request that deflate write the gzip format instead of the zlib - format using deflateInit2(). You can also request that inflate decode the - gzip format using inflateInit2(). Read zlib.h for more details. - -21. Is zlib thread-safe? - - Yes. However any library routines that zlib uses and any application- - provided memory allocation routines must also be thread-safe. zlib's gz* - functions use stdio library routines, and most of zlib's functions use the - library memory allocation routines by default. zlib's *Init* functions - allow for the application to provide custom memory allocation routines. - - Of course, you should only operate on any given zlib or gzip stream from a - single thread at a time. - -22. Can I use zlib in my commercial application? - - Yes. Please read the license in zlib.h. - -23. Is zlib under the GNU license? - - No. Please read the license in zlib.h. - -24. The license says that altered source versions must be "plainly marked". So - what exactly do I need to do to meet that requirement? - - You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In - particular, the final version number needs to be changed to "f", and an - identification string should be appended to ZLIB_VERSION. Version numbers - x.x.x.f are reserved for modifications to zlib by others than the zlib - maintainers. For example, if the version of the base zlib you are altering - is "1.2.3.4", then in zlib.h you should change ZLIB_VERNUM to 0x123f, and - ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also - update the version strings in deflate.c and inftrees.c. - - For altered source distributions, you should also note the origin and - nature of the changes in zlib.h, as well as in ChangeLog and README, along - with the dates of the alterations. The origin should include at least your - name (or your company's name), and an email address to contact for help or - issues with the library. - - Note that distributing a compiled zlib library along with zlib.h and - zconf.h is also a source distribution, and so you should change - ZLIB_VERSION and ZLIB_VERNUM and note the origin and nature of the changes - in zlib.h as you would for a full source distribution. - -25. Will zlib work on a big-endian or little-endian architecture, and can I - exchange compressed data between them? - - Yes and yes. - -26. Will zlib work on a 64-bit machine? - - Yes. It has been tested on 64-bit machines, and has no dependence on any - data types being limited to 32-bits in length. If you have any - difficulties, please provide a complete problem report to zlib@gzip.org - -27. Will zlib decompress data from the PKWare Data Compression Library? - - No. The PKWare DCL uses a completely different compressed data format than - does PKZIP and zlib. However, you can look in zlib's contrib/blast - directory for a possible solution to your problem. - -28. Can I access data randomly in a compressed stream? - - No, not without some preparation. If when compressing you periodically use - Z_FULL_FLUSH, carefully write all the pending data at those points, and - keep an index of those locations, then you can start decompression at those - points. You have to be careful to not use Z_FULL_FLUSH too often, since it - can significantly degrade compression. Alternatively, you can scan a - deflate stream once to generate an index, and then use that index for - random access. See examples/zran.c . - -29. Does zlib work on MVS, OS/390, CICS, etc.? - - It has in the past, but we have not heard of any recent evidence. There - were working ports of zlib 1.1.4 to MVS, but those links no longer work. - If you know of recent, successful applications of zlib on these operating - systems, please let us know. Thanks. - -30. Is there some simpler, easier to read version of inflate I can look at to - understand the deflate format? - - First off, you should read RFC 1951. Second, yes. Look in zlib's - contrib/puff directory. - -31. Does zlib infringe on any patents? - - As far as we know, no. In fact, that was originally the whole point behind - zlib. Look here for some more information: - - http://www.gzip.org/#faq11 - -32. Can zlib work with greater than 4 GB of data? - - Yes. inflate() and deflate() will process any amount of data correctly. - Each call of inflate() or deflate() is limited to input and output chunks - of the maximum value that can be stored in the compiler's "unsigned int" - type, but there is no limit to the number of chunks. Note however that the - strm.total_in and strm_total_out counters may be limited to 4 GB. These - counters are provided as a convenience and are not used internally by - inflate() or deflate(). The application can easily set up its own counters - updated after each call of inflate() or deflate() to count beyond 4 GB. - compress() and uncompress() may be limited to 4 GB, since they operate in a - single call. gzseek() and gztell() may be limited to 4 GB depending on how - zlib is compiled. See the zlibCompileFlags() function in zlib.h. - - The word "may" appears several times above since there is a 4 GB limit only - if the compiler's "long" type is 32 bits. If the compiler's "long" type is - 64 bits, then the limit is 16 exabytes. - -33. Does zlib have any security vulnerabilities? - - The only one that we are aware of is potentially in gzprintf(). If zlib is - compiled to use sprintf() or vsprintf(), then there is no protection - against a buffer overflow of an 8K string space (or other value as set by - gzbuffer()), other than the caller of gzprintf() assuring that the output - will not exceed 8K. On the other hand, if zlib is compiled to use - snprintf() or vsnprintf(), which should normally be the case, then there is - no vulnerability. The ./configure script will display warnings if an - insecure variation of sprintf() will be used by gzprintf(). Also the - zlibCompileFlags() function will return information on what variant of - sprintf() is used by gzprintf(). - - If you don't have snprintf() or vsnprintf() and would like one, you can - find a portable implementation here: - - http://www.ijs.si/software/snprintf/ - - Note that you should be using the most recent version of zlib. Versions - 1.1.3 and before were subject to a double-free vulnerability, and versions - 1.2.1 and 1.2.2 were subject to an access exception when decompressing - invalid compressed data. - -34. Is there a Java version of zlib? - - Probably what you want is to use zlib in Java. zlib is already included - as part of the Java SDK in the java.util.zip package. If you really want - a version of zlib written in the Java language, look on the zlib home - page for links: http://zlib.net/ . - -35. I get this or that compiler or source-code scanner warning when I crank it - up to maximally-pedantic. Can't you guys write proper code? - - Many years ago, we gave up attempting to avoid warnings on every compiler - in the universe. It just got to be a waste of time, and some compilers - were downright silly as well as contradicted each other. So now, we simply - make sure that the code always works. - -36. Valgrind (or some similar memory access checker) says that deflate is - performing a conditional jump that depends on an uninitialized value. - Isn't that a bug? - - No. That is intentional for performance reasons, and the output of deflate - is not affected. This only started showing up recently since zlib 1.2.x - uses malloc() by default for allocations, whereas earlier versions used - calloc(), which zeros out the allocated memory. Even though the code was - correct, versions 1.2.4 and later was changed to not stimulate these - checkers. - -37. Will zlib read the (insert any ancient or arcane format here) compressed - data format? - - Probably not. Look in the comp.compression FAQ for pointers to various - formats and associated software. - -38. How can I encrypt/decrypt zip files with zlib? - - zlib doesn't support encryption. The original PKZIP encryption is very - weak and can be broken with freely available programs. To get strong - encryption, use GnuPG, http://www.gnupg.org/ , which already includes zlib - compression. For PKZIP compatible "encryption", look at - http://www.info-zip.org/ - -39. What's the difference between the "gzip" and "deflate" HTTP 1.1 encodings? - - "gzip" is the gzip format, and "deflate" is the zlib format. They should - probably have called the second one "zlib" instead to avoid confusion with - the raw deflate compressed data format. While the HTTP 1.1 RFC 2616 - correctly points to the zlib specification in RFC 1950 for the "deflate" - transfer encoding, there have been reports of servers and browsers that - incorrectly produce or expect raw deflate data per the deflate - specification in RFC 1951, most notably Microsoft. So even though the - "deflate" transfer encoding using the zlib format would be the more - efficient approach (and in fact exactly what the zlib format was designed - for), using the "gzip" transfer encoding is probably more reliable due to - an unfortunate choice of name on the part of the HTTP 1.1 authors. - - Bottom line: use the gzip format for HTTP 1.1 encoding. - -40. Does zlib support the new "Deflate64" format introduced by PKWare? - - No. PKWare has apparently decided to keep that format proprietary, since - they have not documented it as they have previous compression formats. In - any case, the compression improvements are so modest compared to other more - modern approaches, that it's not worth the effort to implement. - -41. I'm having a problem with the zip functions in zlib, can you help? - - There are no zip functions in zlib. You are probably using minizip by - Giles Vollant, which is found in the contrib directory of zlib. It is not - part of zlib. In fact none of the stuff in contrib is part of zlib. The - files in there are not supported by the zlib authors. You need to contact - the authors of the respective contribution for help. - -42. The match.asm code in contrib is under the GNU General Public License. - Since it's part of zlib, doesn't that mean that all of zlib falls under the - GNU GPL? - - No. The files in contrib are not part of zlib. They were contributed by - other authors and are provided as a convenience to the user within the zlib - distribution. Each item in contrib has its own license. - -43. Is zlib subject to export controls? What is its ECCN? - - zlib is not subject to export controls, and so is classified as EAR99. - -44. Can you please sign these lengthy legal documents and fax them back to us - so that we can use your software in our product? - - No. Go away. Shoo. diff --git a/contrib/zlib/INDEX b/contrib/zlib/INDEX deleted file mode 100644 index 2ba064120..000000000 --- a/contrib/zlib/INDEX +++ /dev/null @@ -1,68 +0,0 @@ -CMakeLists.txt cmake build file -ChangeLog history of changes -FAQ Frequently Asked Questions about zlib -INDEX this file -Makefile dummy Makefile that tells you to ./configure -Makefile.in template for Unix Makefile -README guess what -configure configure script for Unix -make_vms.com makefile for VMS -test/example.c zlib usages examples for build testing -test/minigzip.c minimal gzip-like functionality for build testing -test/infcover.c inf*.c code coverage for build coverage testing -treebuild.xml XML description of source file dependencies -zconf.h.cmakein zconf.h template for cmake -zconf.h.in zconf.h template for configure -zlib.3 Man page for zlib -zlib.3.pdf Man page in PDF format -zlib.map Linux symbol information -zlib.pc.in Template for pkg-config descriptor -zlib.pc.cmakein zlib.pc template for cmake -zlib2ansi perl script to convert source files for C++ compilation - -amiga/ makefiles for Amiga SAS C -as400/ makefiles for AS/400 -doc/ documentation for formats and algorithms -msdos/ makefiles for MSDOS -nintendods/ makefile for Nintendo DS -old/ makefiles for various architectures and zlib documentation - files that have not yet been updated for zlib 1.2.x -qnx/ makefiles for QNX -watcom/ makefiles for OpenWatcom -win32/ makefiles for Windows - - zlib public header files (required for library use): -zconf.h -zlib.h - - private source files used to build the zlib library: -adler32.c -compress.c -crc32.c -crc32.h -deflate.c -deflate.h -gzclose.c -gzguts.h -gzlib.c -gzread.c -gzwrite.c -infback.c -inffast.c -inffast.h -inffixed.h -inflate.c -inflate.h -inftrees.c -inftrees.h -trees.c -trees.h -uncompr.c -zutil.c -zutil.h - - source files for sample programs -See examples/README.examples - - unsupported contributions by third parties -See contrib/README.contrib diff --git a/contrib/zlib/Makefile.in b/contrib/zlib/Makefile.in deleted file mode 100644 index 3413266ae..000000000 --- a/contrib/zlib/Makefile.in +++ /dev/null @@ -1,410 +0,0 @@ -# Makefile for zlib -# Copyright (C) 1995-2017 Jean-loup Gailly, Mark Adler -# For conditions of distribution and use, see copyright notice in zlib.h - -# To compile and test, type: -# ./configure; make test -# Normally configure builds both a static and a shared library. -# If you want to build just a static library, use: ./configure --static - -# To use the asm code, type: -# cp contrib/asm?86/match.S ./match.S -# make LOC=-DASMV OBJA=match.o - -# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type: -# make install -# To install in $HOME instead of /usr/local, use: -# make install prefix=$HOME - -CC=cc - -CFLAGS=-O -#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 -#CFLAGS=-g -DZLIB_DEBUG -#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ -# -Wstrict-prototypes -Wmissing-prototypes - -SFLAGS=-O -LDFLAGS= -TEST_LDFLAGS=-L. libz.a -LDSHARED=$(CC) -CPP=$(CC) -E - -STATICLIB=libz.a -SHAREDLIB=libz.so -SHAREDLIBV=libz.so.1.2.11.1 -SHAREDLIBM=libz.so.1 -LIBS=$(STATICLIB) $(SHAREDLIBV) - -AR=ar -ARFLAGS=rc -RANLIB=ranlib -LDCONFIG=ldconfig -LDSHAREDLIBC=-lc -TAR=tar -SHELL=/bin/sh -EXE= - -prefix = /usr/local -exec_prefix = ${prefix} -libdir = ${exec_prefix}/lib -sharedlibdir = ${libdir} -includedir = ${prefix}/include -mandir = ${prefix}/share/man -man3dir = ${mandir}/man3 -pkgconfigdir = ${libdir}/pkgconfig -SRCDIR= -ZINC= -ZINCOUT=-I. - -OBJZ = adler32.o crc32.o deflate.o infback.o inffast.o inflate.o inftrees.o trees.o zutil.o -OBJG = compress.o uncompr.o gzclose.o gzlib.o gzread.o gzwrite.o -OBJC = $(OBJZ) $(OBJG) - -PIC_OBJZ = adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo -PIC_OBJG = compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo -PIC_OBJC = $(PIC_OBJZ) $(PIC_OBJG) - -# to use the asm code: make OBJA=match.o, PIC_OBJA=match.lo -OBJA = -PIC_OBJA = - -OBJS = $(OBJC) $(OBJA) - -PIC_OBJS = $(PIC_OBJC) $(PIC_OBJA) - -all: static shared - -static: example$(EXE) minigzip$(EXE) - -shared: examplesh$(EXE) minigzipsh$(EXE) - -all64: example64$(EXE) minigzip64$(EXE) - -check: test - -test: all teststatic testshared - -teststatic: static - @TMPST=tmpst_$$; \ - if echo hello world | ./minigzip | ./minigzip -d && ./example $$TMPST ; then \ - echo ' *** zlib test OK ***'; \ - else \ - echo ' *** zlib test FAILED ***'; false; \ - fi - @rm -f tmpst_$$ - -testshared: shared - @LD_LIBRARY_PATH=`pwd`:$(LD_LIBRARY_PATH) ; export LD_LIBRARY_PATH; \ - LD_LIBRARYN32_PATH=`pwd`:$(LD_LIBRARYN32_PATH) ; export LD_LIBRARYN32_PATH; \ - DYLD_LIBRARY_PATH=`pwd`:$(DYLD_LIBRARY_PATH) ; export DYLD_LIBRARY_PATH; \ - SHLIB_PATH=`pwd`:$(SHLIB_PATH) ; export SHLIB_PATH; \ - TMPSH=tmpsh_$$; \ - if echo hello world | ./minigzipsh | ./minigzipsh -d && ./examplesh $$TMPSH; then \ - echo ' *** zlib shared test OK ***'; \ - else \ - echo ' *** zlib shared test FAILED ***'; false; \ - fi - @rm -f tmpsh_$$ - -test64: all64 - @TMP64=tmp64_$$; \ - if echo hello world | ./minigzip64 | ./minigzip64 -d && ./example64 $$TMP64; then \ - echo ' *** zlib 64-bit test OK ***'; \ - else \ - echo ' *** zlib 64-bit test FAILED ***'; false; \ - fi - @rm -f tmp64_$$ - -infcover.o: $(SRCDIR)test/infcover.c $(SRCDIR)zlib.h zconf.h - $(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/infcover.c - -infcover: infcover.o libz.a - $(CC) $(CFLAGS) -o $@ infcover.o libz.a - -cover: infcover - rm -f *.gcda - ./infcover - gcov inf*.c - -libz.a: $(OBJS) - $(AR) $(ARFLAGS) $@ $(OBJS) - -@ ($(RANLIB) $@ || true) >/dev/null 2>&1 - -match.o: match.S - $(CPP) match.S > _match.s - $(CC) -c _match.s - mv _match.o match.o - rm -f _match.s - -match.lo: match.S - $(CPP) match.S > _match.s - $(CC) -c -fPIC _match.s - mv _match.o match.lo - rm -f _match.s - -example.o: $(SRCDIR)test/example.c $(SRCDIR)zlib.h zconf.h - $(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/example.c - -minigzip.o: $(SRCDIR)test/minigzip.c $(SRCDIR)zlib.h zconf.h - $(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/minigzip.c - -example64.o: $(SRCDIR)test/example.c $(SRCDIR)zlib.h zconf.h - $(CC) $(CFLAGS) $(ZINCOUT) -D_FILE_OFFSET_BITS=64 -c -o $@ $(SRCDIR)test/example.c - -minigzip64.o: $(SRCDIR)test/minigzip.c $(SRCDIR)zlib.h zconf.h - $(CC) $(CFLAGS) $(ZINCOUT) -D_FILE_OFFSET_BITS=64 -c -o $@ $(SRCDIR)test/minigzip.c - - -adler32.o: $(SRCDIR)adler32.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)adler32.c - -crc32.o: $(SRCDIR)crc32.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)crc32.c - -deflate.o: $(SRCDIR)deflate.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)deflate.c - -infback.o: $(SRCDIR)infback.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)infback.c - -inffast.o: $(SRCDIR)inffast.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)inffast.c - -inflate.o: $(SRCDIR)inflate.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)inflate.c - -inftrees.o: $(SRCDIR)inftrees.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)inftrees.c - -trees.o: $(SRCDIR)trees.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)trees.c - -zutil.o: $(SRCDIR)zutil.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)zutil.c - -compress.o: $(SRCDIR)compress.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)compress.c - -uncompr.o: $(SRCDIR)uncompr.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)uncompr.c - -gzclose.o: $(SRCDIR)gzclose.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzclose.c - -gzlib.o: $(SRCDIR)gzlib.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzlib.c - -gzread.o: $(SRCDIR)gzread.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzread.c - -gzwrite.o: $(SRCDIR)gzwrite.c - $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)gzwrite.c - - -adler32.lo: $(SRCDIR)adler32.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/adler32.o $(SRCDIR)adler32.c - -@mv objs/adler32.o $@ - -crc32.lo: $(SRCDIR)crc32.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/crc32.o $(SRCDIR)crc32.c - -@mv objs/crc32.o $@ - -deflate.lo: $(SRCDIR)deflate.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/deflate.o $(SRCDIR)deflate.c - -@mv objs/deflate.o $@ - -infback.lo: $(SRCDIR)infback.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/infback.o $(SRCDIR)infback.c - -@mv objs/infback.o $@ - -inffast.lo: $(SRCDIR)inffast.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/inffast.o $(SRCDIR)inffast.c - -@mv objs/inffast.o $@ - -inflate.lo: $(SRCDIR)inflate.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/inflate.o $(SRCDIR)inflate.c - -@mv objs/inflate.o $@ - -inftrees.lo: $(SRCDIR)inftrees.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/inftrees.o $(SRCDIR)inftrees.c - -@mv objs/inftrees.o $@ - -trees.lo: $(SRCDIR)trees.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/trees.o $(SRCDIR)trees.c - -@mv objs/trees.o $@ - -zutil.lo: $(SRCDIR)zutil.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/zutil.o $(SRCDIR)zutil.c - -@mv objs/zutil.o $@ - -compress.lo: $(SRCDIR)compress.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/compress.o $(SRCDIR)compress.c - -@mv objs/compress.o $@ - -uncompr.lo: $(SRCDIR)uncompr.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/uncompr.o $(SRCDIR)uncompr.c - -@mv objs/uncompr.o $@ - -gzclose.lo: $(SRCDIR)gzclose.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzclose.o $(SRCDIR)gzclose.c - -@mv objs/gzclose.o $@ - -gzlib.lo: $(SRCDIR)gzlib.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzlib.o $(SRCDIR)gzlib.c - -@mv objs/gzlib.o $@ - -gzread.lo: $(SRCDIR)gzread.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzread.o $(SRCDIR)gzread.c - -@mv objs/gzread.o $@ - -gzwrite.lo: $(SRCDIR)gzwrite.c - -@mkdir objs 2>/dev/null || test -d objs - $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/gzwrite.o $(SRCDIR)gzwrite.c - -@mv objs/gzwrite.o $@ - - -placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a - $(LDSHARED) $(SFLAGS) -o $@ $(PIC_OBJS) $(LDSHAREDLIBC) $(LDFLAGS) - rm -f $(SHAREDLIB) $(SHAREDLIBM) - ln -s $@ $(SHAREDLIB) - ln -s $@ $(SHAREDLIBM) - -@rmdir objs - -example$(EXE): example.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ example.o $(TEST_LDFLAGS) - -minigzip$(EXE): minigzip.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ minigzip.o $(TEST_LDFLAGS) - -examplesh$(EXE): example.o $(SHAREDLIBV) - $(CC) $(CFLAGS) -o $@ example.o -L. $(SHAREDLIBV) - -minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) - $(CC) $(CFLAGS) -o $@ minigzip.o -L. $(SHAREDLIBV) - -example64$(EXE): example64.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ example64.o $(TEST_LDFLAGS) - -minigzip64$(EXE): minigzip64.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ minigzip64.o $(TEST_LDFLAGS) - -install-libs: $(LIBS) - -@if [ ! -d $(DESTDIR)$(exec_prefix) ]; then mkdir -p $(DESTDIR)$(exec_prefix); fi - -@if [ ! -d $(DESTDIR)$(libdir) ]; then mkdir -p $(DESTDIR)$(libdir); fi - -@if [ ! -d $(DESTDIR)$(sharedlibdir) ]; then mkdir -p $(DESTDIR)$(sharedlibdir); fi - -@if [ ! -d $(DESTDIR)$(man3dir) ]; then mkdir -p $(DESTDIR)$(man3dir); fi - -@if [ ! -d $(DESTDIR)$(pkgconfigdir) ]; then mkdir -p $(DESTDIR)$(pkgconfigdir); fi - rm -f $(DESTDIR)$(libdir)/$(STATICLIB) - cp $(STATICLIB) $(DESTDIR)$(libdir) - chmod 644 $(DESTDIR)$(libdir)/$(STATICLIB) - -@($(RANLIB) $(DESTDIR)$(libdir)/libz.a || true) >/dev/null 2>&1 - -@if test -n "$(SHAREDLIBV)"; then \ - rm -f $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV); \ - cp $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir); \ - echo "cp $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)"; \ - chmod 755 $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV); \ - echo "chmod 755 $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV)"; \ - rm -f $(DESTDIR)$(sharedlibdir)/$(SHAREDLIB) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBM); \ - ln -s $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIB); \ - ln -s $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBM); \ - ($(LDCONFIG) || true) >/dev/null 2>&1; \ - fi - rm -f $(DESTDIR)$(man3dir)/zlib.3 - cp $(SRCDIR)zlib.3 $(DESTDIR)$(man3dir) - chmod 644 $(DESTDIR)$(man3dir)/zlib.3 - rm -f $(DESTDIR)$(pkgconfigdir)/zlib.pc - cp zlib.pc $(DESTDIR)$(pkgconfigdir) - chmod 644 $(DESTDIR)$(pkgconfigdir)/zlib.pc -# The ranlib in install is needed on NeXTSTEP which checks file times -# ldconfig is for Linux - -install: install-libs - -@if [ ! -d $(DESTDIR)$(includedir) ]; then mkdir -p $(DESTDIR)$(includedir); fi - rm -f $(DESTDIR)$(includedir)/zlib.h $(DESTDIR)$(includedir)/zconf.h - cp $(SRCDIR)zlib.h zconf.h $(DESTDIR)$(includedir) - chmod 644 $(DESTDIR)$(includedir)/zlib.h $(DESTDIR)$(includedir)/zconf.h - -uninstall: - cd $(DESTDIR)$(includedir) && rm -f zlib.h zconf.h - cd $(DESTDIR)$(libdir) && rm -f libz.a; \ - if test -n "$(SHAREDLIBV)" -a -f $(SHAREDLIBV); then \ - rm -f $(SHAREDLIBV) $(SHAREDLIB) $(SHAREDLIBM); \ - fi - cd $(DESTDIR)$(man3dir) && rm -f zlib.3 - cd $(DESTDIR)$(pkgconfigdir) && rm -f zlib.pc - -docs: zlib.3.pdf - -zlib.3.pdf: $(SRCDIR)zlib.3 - groff -mandoc -f H -T ps $(SRCDIR)zlib.3 | ps2pdf - $@ - -zconf.h.cmakein: $(SRCDIR)zconf.h.in - -@ TEMPFILE=zconfh_$$; \ - echo "/#define ZCONF_H/ a\\\\\n#cmakedefine Z_PREFIX\\\\\n#cmakedefine Z_HAVE_UNISTD_H\n" >> $$TEMPFILE &&\ - sed -f $$TEMPFILE $(SRCDIR)zconf.h.in > $@ &&\ - touch -r $(SRCDIR)zconf.h.in $@ &&\ - rm $$TEMPFILE - -zconf: $(SRCDIR)zconf.h.in - cp -p $(SRCDIR)zconf.h.in zconf.h - -mostlyclean: clean -clean: - rm -f *.o *.lo *~ \ - example$(EXE) minigzip$(EXE) examplesh$(EXE) minigzipsh$(EXE) \ - example64$(EXE) minigzip64$(EXE) \ - infcover \ - libz.* foo.gz so_locations \ - _match.s maketree contrib/infback9/*.o - rm -rf objs - rm -f *.gcda *.gcno *.gcov - rm -f contrib/infback9/*.gcda contrib/infback9/*.gcno contrib/infback9/*.gcov - -maintainer-clean: distclean -distclean: clean zconf zconf.h.cmakein docs - rm -f Makefile zlib.pc configure.log - -@rm -f .DS_Store - @if [ -f Makefile.in ]; then \ - printf 'all:\n\t-@echo "Please use ./configure first. Thank you."\n' > Makefile ; \ - printf '\ndistclean:\n\tmake -f Makefile.in distclean\n' >> Makefile ; \ - touch -r $(SRCDIR)Makefile.in Makefile ; fi - @if [ ! -f zconf.h.in ]; then rm -f zconf.h zconf.h.cmakein ; fi - @if [ ! -f zlib.3 ]; then rm -f zlib.3.pdf ; fi - -tags: - etags $(SRCDIR)*.[ch] - -adler32.o zutil.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h -gzclose.o gzlib.o gzread.o gzwrite.o: $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h -compress.o example.o minigzip.o uncompr.o: $(SRCDIR)zlib.h zconf.h -crc32.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)crc32.h -deflate.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h -infback.o inflate.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h $(SRCDIR)inffixed.h -inffast.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h -inftrees.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h -trees.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h - -adler32.lo zutil.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h -gzclose.lo gzlib.lo gzread.lo gzwrite.lo: $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h -compress.lo example.lo minigzip.lo uncompr.lo: $(SRCDIR)zlib.h zconf.h -crc32.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)crc32.h -deflate.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h -infback.lo inflate.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h $(SRCDIR)inffixed.h -inffast.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h -inftrees.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h -trees.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h diff --git a/contrib/zlib/configure b/contrib/zlib/configure deleted file mode 100644 index f4127954c..000000000 --- a/contrib/zlib/configure +++ /dev/null @@ -1,921 +0,0 @@ -#!/bin/sh -# configure script for zlib. -# -# Normally configure builds both a static and a shared library. -# If you want to build just a static library, use: ./configure --static -# -# To impose specific compiler or flags or install directory, use for example: -# prefix=$HOME CC=cc CFLAGS="-O4" ./configure -# or for csh/tcsh users: -# (setenv prefix $HOME; setenv CC cc; setenv CFLAGS "-O4"; ./configure) - -# Incorrect settings of CC or CFLAGS may prevent creating a shared library. -# If you have problems, try without defining CC and CFLAGS before reporting -# an error. - -# start off configure.log -echo -------------------- >> configure.log -echo $0 $* >> configure.log -date >> configure.log - -# get source directory -SRCDIR=`dirname $0` -if test $SRCDIR = "."; then - ZINC="" - ZINCOUT="-I." - SRCDIR="" -else - ZINC='-include zconf.h' - ZINCOUT='-I. -I$(SRCDIR)' - SRCDIR="$SRCDIR/" -fi - -# set command prefix for cross-compilation -if [ -n "${CHOST}" ]; then - uname="`echo "${CHOST}" | sed -e 's/^[^-]*-\([^-]*\)$/\1/' -e 's/^[^-]*-[^-]*-\([^-]*\)$/\1/' -e 's/^[^-]*-[^-]*-\([^-]*\)-.*$/\1/'`" - CROSS_PREFIX="${CHOST}-" -fi - -# destination name for static library -STATICLIB=libz.a - -# extract zlib version numbers from zlib.h -VER=`sed -n -e '/VERSION "/s/.*"\(.*\)".*/\1/p' < ${SRCDIR}zlib.h` -VER3=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\\.[0-9]*\).*/\1/p' < ${SRCDIR}zlib.h` -VER2=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` -VER1=`sed -n -e '/VERSION "/s/.*"\([0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` - -# establish commands for library building -if "${CROSS_PREFIX}ar" --version >/dev/null 2>/dev/null || test $? -lt 126; then - AR=${AR-"${CROSS_PREFIX}ar"} - test -n "${CROSS_PREFIX}" && echo Using ${AR} | tee -a configure.log -else - AR=${AR-"ar"} - test -n "${CROSS_PREFIX}" && echo Using ${AR} | tee -a configure.log -fi -ARFLAGS=${ARFLAGS-"rc"} -if "${CROSS_PREFIX}ranlib" --version >/dev/null 2>/dev/null || test $? -lt 126; then - RANLIB=${RANLIB-"${CROSS_PREFIX}ranlib"} - test -n "${CROSS_PREFIX}" && echo Using ${RANLIB} | tee -a configure.log -else - RANLIB=${RANLIB-"ranlib"} -fi -if "${CROSS_PREFIX}nm" --version >/dev/null 2>/dev/null || test $? -lt 126; then - NM=${NM-"${CROSS_PREFIX}nm"} - test -n "${CROSS_PREFIX}" && echo Using ${NM} | tee -a configure.log -else - NM=${NM-"nm"} -fi - -# set defaults before processing command line options -LDCONFIG=${LDCONFIG-"ldconfig"} -LDSHAREDLIBC="${LDSHAREDLIBC--lc}" -ARCHS= -prefix=${prefix-/usr/local} -exec_prefix=${exec_prefix-'${prefix}'} -libdir=${libdir-'${exec_prefix}/lib'} -sharedlibdir=${sharedlibdir-'${libdir}'} -includedir=${includedir-'${prefix}/include'} -mandir=${mandir-'${prefix}/share/man'} -shared_ext='.so' -shared=1 -solo=0 -cover=0 -zprefix=0 -zconst=0 -build64=0 -gcc=0 -warn=0 -debug=0 -old_cc="$CC" -old_cflags="$CFLAGS" -OBJC='$(OBJZ) $(OBJG)' -PIC_OBJC='$(PIC_OBJZ) $(PIC_OBJG)' - -# leave this script, optionally in a bad way -leave() -{ - if test "$*" != "0"; then - echo "** $0 aborting." | tee -a configure.log - fi - rm -f $test.[co] $test $test$shared_ext $test.gcno ./--version - echo -------------------- >> configure.log - echo >> configure.log - echo >> configure.log - exit $1 -} - -# process command line options -while test $# -ge 1 -do -case "$1" in - -h* | --help) - echo 'usage:' | tee -a configure.log - echo ' configure [--const] [--zprefix] [--prefix=PREFIX] [--eprefix=EXPREFIX]' | tee -a configure.log - echo ' [--static] [--64] [--libdir=LIBDIR] [--sharedlibdir=LIBDIR]' | tee -a configure.log - echo ' [--includedir=INCLUDEDIR] [--archs="-arch i386 -arch x86_64"]' | tee -a configure.log - exit 0 ;; - -p*=* | --prefix=*) prefix=`echo $1 | sed 's/.*=//'`; shift ;; - -e*=* | --eprefix=*) exec_prefix=`echo $1 | sed 's/.*=//'`; shift ;; - -l*=* | --libdir=*) libdir=`echo $1 | sed 's/.*=//'`; shift ;; - --sharedlibdir=*) sharedlibdir=`echo $1 | sed 's/.*=//'`; shift ;; - -i*=* | --includedir=*) includedir=`echo $1 | sed 's/.*=//'`;shift ;; - -u*=* | --uname=*) uname=`echo $1 | sed 's/.*=//'`;shift ;; - -p* | --prefix) prefix="$2"; shift; shift ;; - -e* | --eprefix) exec_prefix="$2"; shift; shift ;; - -l* | --libdir) libdir="$2"; shift; shift ;; - -i* | --includedir) includedir="$2"; shift; shift ;; - -s* | --shared | --enable-shared) shared=1; shift ;; - -t | --static) shared=0; shift ;; - --solo) solo=1; shift ;; - --cover) cover=1; shift ;; - -z* | --zprefix) zprefix=1; shift ;; - -6* | --64) build64=1; shift ;; - -a*=* | --archs=*) ARCHS=`echo $1 | sed 's/.*=//'`; shift ;; - --sysconfdir=*) echo "ignored option: --sysconfdir" | tee -a configure.log; shift ;; - --localstatedir=*) echo "ignored option: --localstatedir" | tee -a configure.log; shift ;; - -c* | --const) zconst=1; shift ;; - -w* | --warn) warn=1; shift ;; - -d* | --debug) debug=1; shift ;; - *) - echo "unknown option: $1" | tee -a configure.log - echo "$0 --help for help" | tee -a configure.log - leave 1;; - esac -done - -# temporary file name -test=ztest$$ - -# put arguments in log, also put test file in log if used in arguments -show() -{ - case "$*" in - *$test.c*) - echo === $test.c === >> configure.log - cat $test.c >> configure.log - echo === >> configure.log;; - esac - echo $* >> configure.log -} - -# check for gcc vs. cc and set compile and link flags based on the system identified by uname -cat > $test.c <&1` in - *gcc*) gcc=1 ;; - *clang*) gcc=1 ;; -esac - -show $cc -c $test.c -if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then - echo ... using gcc >> configure.log - CC="$cc" - CFLAGS="${CFLAGS--O3}" - SFLAGS="${CFLAGS--O3} -fPIC" - if test "$ARCHS"; then - CFLAGS="${CFLAGS} ${ARCHS}" - LDFLAGS="${LDFLAGS} ${ARCHS}" - fi - if test $build64 -eq 1; then - CFLAGS="${CFLAGS} -m64" - SFLAGS="${SFLAGS} -m64" - fi - if test "$warn" -eq 1; then - if test "$zconst" -eq 1; then - CFLAGS="${CFLAGS} -Wall -Wextra -Wcast-qual -pedantic -DZLIB_CONST -fsanitize=address" - else - CFLAGS="${CFLAGS} -Wall -Wextra -pedantic -fsanitize=address" - fi - fi - if test $debug -eq 1; then - CFLAGS="${CFLAGS} -DZLIB_DEBUG" - SFLAGS="${SFLAGS} -DZLIB_DEBUG" - fi - if test -z "$uname"; then - uname=`(uname -s || echo unknown) 2>/dev/null` - fi - case "$uname" in - Linux* | linux* | GNU | GNU/* | solaris*) - LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1,--version-script,${SRCDIR}zlib.map"} ;; - *BSD | *bsd* | DragonFly) - LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1,--version-script,${SRCDIR}zlib.map"} - LDCONFIG="ldconfig -m" ;; - CYGWIN* | Cygwin* | cygwin* | OS/2*) - EXE='.exe' ;; - MINGW* | mingw*) -# temporary bypass - rm -f $test.[co] $test $test$shared_ext - echo "Please use win32/Makefile.gcc instead." | tee -a configure.log - leave 1 - LDSHARED=${LDSHARED-"$cc -shared"} - LDSHAREDLIBC="" - EXE='.exe' ;; - QNX*) # This is for QNX6. I suppose that the QNX rule below is for QNX2,QNX4 - # (alain.bonnefoy@icbt.com) - LDSHARED=${LDSHARED-"$cc -shared -Wl,-hlibz.so.1"} ;; - HP-UX*) - LDSHARED=${LDSHARED-"$cc -shared $SFLAGS"} - case `(uname -m || echo unknown) 2>/dev/null` in - ia64) - shared_ext='.so' - SHAREDLIB='libz.so' ;; - *) - shared_ext='.sl' - SHAREDLIB='libz.sl' ;; - esac ;; - Darwin* | darwin*) - shared_ext='.dylib' - SHAREDLIB=libz$shared_ext - SHAREDLIBV=libz.$VER$shared_ext - SHAREDLIBM=libz.$VER1$shared_ext - LDSHARED=${LDSHARED-"$cc -dynamiclib -install_name $libdir/$SHAREDLIBM -compatibility_version $VER1 -current_version $VER3"} - if libtool -V 2>&1 | grep Apple > /dev/null; then - AR="libtool" - else - AR="/usr/bin/libtool" - fi - ARFLAGS="-o" ;; - *) LDSHARED=${LDSHARED-"$cc -shared"} ;; - esac -else - # find system name and corresponding cc options - CC=${CC-cc} - gcc=0 - echo ... using $CC >> configure.log - if test -z "$uname"; then - uname=`(uname -sr || echo unknown) 2>/dev/null` - fi - case "$uname" in - HP-UX*) SFLAGS=${CFLAGS-"-O +z"} - CFLAGS=${CFLAGS-"-O"} -# LDSHARED=${LDSHARED-"ld -b +vnocompatwarnings"} - LDSHARED=${LDSHARED-"ld -b"} - case `(uname -m || echo unknown) 2>/dev/null` in - ia64) - shared_ext='.so' - SHAREDLIB='libz.so' ;; - *) - shared_ext='.sl' - SHAREDLIB='libz.sl' ;; - esac ;; - IRIX*) SFLAGS=${CFLAGS-"-ansi -O2 -rpath ."} - CFLAGS=${CFLAGS-"-ansi -O2"} - LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so.1"} ;; - OSF1\ V4*) SFLAGS=${CFLAGS-"-O -std1"} - CFLAGS=${CFLAGS-"-O -std1"} - LDFLAGS="${LDFLAGS} -Wl,-rpath,." - LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so -Wl,-msym -Wl,-rpath,$(libdir) -Wl,-set_version,${VER}:1.0"} ;; - OSF1*) SFLAGS=${CFLAGS-"-O -std1"} - CFLAGS=${CFLAGS-"-O -std1"} - LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so.1"} ;; - QNX*) SFLAGS=${CFLAGS-"-4 -O"} - CFLAGS=${CFLAGS-"-4 -O"} - LDSHARED=${LDSHARED-"cc"} - RANLIB=${RANLIB-"true"} - AR="cc" - ARFLAGS="-A" ;; - SCO_SV\ 3.2*) SFLAGS=${CFLAGS-"-O3 -dy -KPIC "} - CFLAGS=${CFLAGS-"-O3"} - LDSHARED=${LDSHARED-"cc -dy -KPIC -G"} ;; - SunOS\ 5* | solaris*) - LDSHARED=${LDSHARED-"cc -G -h libz$shared_ext.$VER1"} - SFLAGS=${CFLAGS-"-fast -KPIC"} - CFLAGS=${CFLAGS-"-fast"} - if test $build64 -eq 1; then - # old versions of SunPRO/Workshop/Studio don't support -m64, - # but newer ones do. Check for it. - flag64=`$CC -flags | egrep -- '^-m64'` - if test x"$flag64" != x"" ; then - CFLAGS="${CFLAGS} -m64" - SFLAGS="${SFLAGS} -m64" - else - case `(uname -m || echo unknown) 2>/dev/null` in - i86*) - SFLAGS="$SFLAGS -xarch=amd64" - CFLAGS="$CFLAGS -xarch=amd64" ;; - *) - SFLAGS="$SFLAGS -xarch=v9" - CFLAGS="$CFLAGS -xarch=v9" ;; - esac - fi - fi - if test -n "$ZINC"; then - ZINC='-I- -I. -I$(SRCDIR)' - fi - ;; - SunOS\ 4*) SFLAGS=${CFLAGS-"-O2 -PIC"} - CFLAGS=${CFLAGS-"-O2"} - LDSHARED=${LDSHARED-"ld"} ;; - SunStudio\ 9*) SFLAGS=${CFLAGS-"-fast -xcode=pic32 -xtarget=ultra3 -xarch=v9b"} - CFLAGS=${CFLAGS-"-fast -xtarget=ultra3 -xarch=v9b"} - LDSHARED=${LDSHARED-"cc -xarch=v9b"} ;; - UNIX_System_V\ 4.2.0) - SFLAGS=${CFLAGS-"-KPIC -O"} - CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -G"} ;; - UNIX_SV\ 4.2MP) - SFLAGS=${CFLAGS-"-Kconform_pic -O"} - CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -G"} ;; - OpenUNIX\ 5) - SFLAGS=${CFLAGS-"-KPIC -O"} - CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -G"} ;; - AIX*) # Courtesy of dbakker@arrayasolutions.com - SFLAGS=${CFLAGS-"-O -qmaxmem=8192"} - CFLAGS=${CFLAGS-"-O -qmaxmem=8192"} - LDSHARED=${LDSHARED-"xlc -G"} ;; - # send working options for other systems to zlib@gzip.org - *) SFLAGS=${CFLAGS-"-O"} - CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -shared"} ;; - esac -fi - -# destination names for shared library if not defined above -SHAREDLIB=${SHAREDLIB-"libz$shared_ext"} -SHAREDLIBV=${SHAREDLIBV-"libz$shared_ext.$VER"} -SHAREDLIBM=${SHAREDLIBM-"libz$shared_ext.$VER1"} - -echo >> configure.log - -# define functions for testing compiler and library characteristics and logging the results - -cat > $test.c </dev/null; then - try() - { - show $* - test "`( $* ) 2>&1 | tee -a configure.log`" = "" - } - echo - using any output from compiler to indicate an error >> configure.log -else - try() - { - show $* - ( $* ) >> configure.log 2>&1 - ret=$? - if test $ret -ne 0; then - echo "(exit code "$ret")" >> configure.log - fi - return $ret - } -fi - -tryboth() -{ - show $* - got=`( $* ) 2>&1` - ret=$? - printf %s "$got" >> configure.log - if test $ret -ne 0; then - return $ret - fi - test "$got" = "" -} - -cat > $test.c << EOF -int foo() { return 0; } -EOF -echo "Checking for obsessive-compulsive compiler options..." >> configure.log -if try $CC -c $CFLAGS $test.c; then - : -else - echo "Compiler error reporting is too harsh for $0 (perhaps remove -Werror)." | tee -a configure.log - leave 1 -fi - -echo >> configure.log - -# see if shared library build supported -cat > $test.c <> configure.log - show "$NM $test.o | grep _hello" - if test "`$NM $test.o | grep _hello | tee -a configure.log`" = ""; then - CPP="$CPP -DNO_UNDERLINE" - echo Checking for underline in external names... No. | tee -a configure.log - else - echo Checking for underline in external names... Yes. | tee -a configure.log - fi ;; -esac - -echo >> configure.log - -# check for size_t -cat > $test.c < -#include -size_t dummy = 0; -EOF -if try $CC -c $CFLAGS $test.c; then - echo "Checking for size_t... Yes." | tee -a configure.log - need_sizet=0 -else - echo "Checking for size_t... No." | tee -a configure.log - need_sizet=1 -fi - -echo >> configure.log - -# find the size_t integer type, if needed -if test $need_sizet -eq 1; then - cat > $test.c < $test.c < -int main(void) { - if (sizeof(void *) <= sizeof(int)) puts("int"); - else if (sizeof(void *) <= sizeof(long)) puts("long"); - else puts("z_longlong"); - return 0; -} -EOF - else - echo "Checking for long long... No." | tee -a configure.log - cat > $test.c < -int main(void) { - if (sizeof(void *) <= sizeof(int)) puts("int"); - else puts("long"); - return 0; -} -EOF - fi - if try $CC $CFLAGS -o $test $test.c; then - sizet=`./$test` - echo "Checking for a pointer-size integer type..." $sizet"." | tee -a configure.log - else - echo "Failed to find a pointer-size integer type." | tee -a configure.log - leave 1 - fi -fi - -if test $need_sizet -eq 1; then - CFLAGS="${CFLAGS} -DNO_SIZE_T=${sizet}" - SFLAGS="${SFLAGS} -DNO_SIZE_T=${sizet}" -fi - -echo >> configure.log - -# check for large file support, and if none, check for fseeko() -cat > $test.c < -off64_t dummy = 0; -EOF -if try $CC -c $CFLAGS -D_LARGEFILE64_SOURCE=1 $test.c; then - CFLAGS="${CFLAGS} -D_LARGEFILE64_SOURCE=1" - SFLAGS="${SFLAGS} -D_LARGEFILE64_SOURCE=1" - ALL="${ALL} all64" - TEST="${TEST} test64" - echo "Checking for off64_t... Yes." | tee -a configure.log - echo "Checking for fseeko... Yes." | tee -a configure.log -else - echo "Checking for off64_t... No." | tee -a configure.log - echo >> configure.log - cat > $test.c < -int main(void) { - fseeko(NULL, 0, 0); - return 0; -} -EOF - if try $CC $CFLAGS -o $test $test.c; then - echo "Checking for fseeko... Yes." | tee -a configure.log - else - CFLAGS="${CFLAGS} -DNO_FSEEKO" - SFLAGS="${SFLAGS} -DNO_FSEEKO" - echo "Checking for fseeko... No." | tee -a configure.log - fi -fi - -echo >> configure.log - -# check for strerror() for use by gz* functions -cat > $test.c < -#include -int main() { return strlen(strerror(errno)); } -EOF -if try $CC $CFLAGS -o $test $test.c; then - echo "Checking for strerror... Yes." | tee -a configure.log -else - CFLAGS="${CFLAGS} -DNO_STRERROR" - SFLAGS="${SFLAGS} -DNO_STRERROR" - echo "Checking for strerror... No." | tee -a configure.log -fi - -# copy clean zconf.h for subsequent edits -cp -p ${SRCDIR}zconf.h.in zconf.h - -echo >> configure.log - -# check for unistd.h and save result in zconf.h -cat > $test.c < -int main() { return 0; } -EOF -if try $CC -c $CFLAGS $test.c; then - sed < zconf.h "/^#ifdef HAVE_UNISTD_H.* may be/s/def HAVE_UNISTD_H\(.*\) may be/ 1\1 was/" > zconf.temp.h - mv zconf.temp.h zconf.h - echo "Checking for unistd.h... Yes." | tee -a configure.log -else - echo "Checking for unistd.h... No." | tee -a configure.log -fi - -echo >> configure.log - -# check for stdarg.h and save result in zconf.h -cat > $test.c < -int main() { return 0; } -EOF -if try $CC -c $CFLAGS $test.c; then - sed < zconf.h "/^#ifdef HAVE_STDARG_H.* may be/s/def HAVE_STDARG_H\(.*\) may be/ 1\1 was/" > zconf.temp.h - mv zconf.temp.h zconf.h - echo "Checking for stdarg.h... Yes." | tee -a configure.log -else - echo "Checking for stdarg.h... No." | tee -a configure.log -fi - -# if the z_ prefix was requested, save that in zconf.h -if test $zprefix -eq 1; then - sed < zconf.h "/#ifdef Z_PREFIX.* may be/s/def Z_PREFIX\(.*\) may be/ 1\1 was/" > zconf.temp.h - mv zconf.temp.h zconf.h - echo >> configure.log - echo "Using z_ prefix on all symbols." | tee -a configure.log -fi - -# if --solo compilation was requested, save that in zconf.h and remove gz stuff from object lists -if test $solo -eq 1; then - sed '/#define ZCONF_H/a\ -#define Z_SOLO - -' < zconf.h > zconf.temp.h - mv zconf.temp.h zconf.h -OBJC='$(OBJZ)' -PIC_OBJC='$(PIC_OBJZ)' -fi - -# if code coverage testing was requested, use older gcc if defined, e.g. "gcc-4.2" on Mac OS X -if test $cover -eq 1; then - CFLAGS="${CFLAGS} -fprofile-arcs -ftest-coverage" - if test -n "$GCC_CLASSIC"; then - CC=$GCC_CLASSIC - fi -fi - -echo >> configure.log - -# conduct a series of tests to resolve eight possible cases of using "vs" or "s" printf functions -# (using stdarg or not), with or without "n" (proving size of buffer), and with or without a -# return value. The most secure result is vsnprintf() with a return value. snprintf() with a -# return value is secure as well, but then gzprintf() will be limited to 20 arguments. -cat > $test.c < -#include -#include "zconf.h" -int main() -{ -#ifndef STDC - choke me -#endif - return 0; -} -EOF -if try $CC -c $CFLAGS $test.c; then - echo "Checking whether to use vs[n]printf() or s[n]printf()... using vs[n]printf()." | tee -a configure.log - - echo >> configure.log - cat > $test.c < -#include -int mytest(const char *fmt, ...) -{ - char buf[20]; - va_list ap; - va_start(ap, fmt); - vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); - return 0; -} -int main() -{ - return (mytest("Hello%d\n", 1)); -} -EOF - if try $CC $CFLAGS -o $test $test.c; then - echo "Checking for vsnprintf() in stdio.h... Yes." | tee -a configure.log - - echo >> configure.log - cat >$test.c < -#include -int mytest(const char *fmt, ...) -{ - int n; - char buf[20]; - va_list ap; - va_start(ap, fmt); - n = vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); - return n; -} -int main() -{ - return (mytest("Hello%d\n", 1)); -} -EOF - - if try $CC -c $CFLAGS $test.c; then - echo "Checking for return value of vsnprintf()... Yes." | tee -a configure.log - else - CFLAGS="$CFLAGS -DHAS_vsnprintf_void" - SFLAGS="$SFLAGS -DHAS_vsnprintf_void" - echo "Checking for return value of vsnprintf()... No." | tee -a configure.log - echo " WARNING: apparently vsnprintf() does not return a value. zlib" | tee -a configure.log - echo " can build but will be open to possible string-format security" | tee -a configure.log - echo " vulnerabilities." | tee -a configure.log - fi - else - CFLAGS="$CFLAGS -DNO_vsnprintf" - SFLAGS="$SFLAGS -DNO_vsnprintf" - echo "Checking for vsnprintf() in stdio.h... No." | tee -a configure.log - echo " WARNING: vsnprintf() not found, falling back to vsprintf(). zlib" | tee -a configure.log - echo " can build but will be open to possible buffer-overflow security" | tee -a configure.log - echo " vulnerabilities." | tee -a configure.log - - echo >> configure.log - cat >$test.c < -#include -int mytest(const char *fmt, ...) -{ - int n; - char buf[20]; - va_list ap; - va_start(ap, fmt); - n = vsprintf(buf, fmt, ap); - va_end(ap); - return n; -} -int main() -{ - return (mytest("Hello%d\n", 1)); -} -EOF - - if try $CC -c $CFLAGS $test.c; then - echo "Checking for return value of vsprintf()... Yes." | tee -a configure.log - else - CFLAGS="$CFLAGS -DHAS_vsprintf_void" - SFLAGS="$SFLAGS -DHAS_vsprintf_void" - echo "Checking for return value of vsprintf()... No." | tee -a configure.log - echo " WARNING: apparently vsprintf() does not return a value. zlib" | tee -a configure.log - echo " can build but will be open to possible string-format security" | tee -a configure.log - echo " vulnerabilities." | tee -a configure.log - fi - fi -else - echo "Checking whether to use vs[n]printf() or s[n]printf()... using s[n]printf()." | tee -a configure.log - - echo >> configure.log - cat >$test.c < -int mytest() -{ - char buf[20]; - snprintf(buf, sizeof(buf), "%s", "foo"); - return 0; -} -int main() -{ - return (mytest()); -} -EOF - - if try $CC $CFLAGS -o $test $test.c; then - echo "Checking for snprintf() in stdio.h... Yes." | tee -a configure.log - - echo >> configure.log - cat >$test.c < -int mytest() -{ - char buf[20]; - return snprintf(buf, sizeof(buf), "%s", "foo"); -} -int main() -{ - return (mytest()); -} -EOF - - if try $CC -c $CFLAGS $test.c; then - echo "Checking for return value of snprintf()... Yes." | tee -a configure.log - else - CFLAGS="$CFLAGS -DHAS_snprintf_void" - SFLAGS="$SFLAGS -DHAS_snprintf_void" - echo "Checking for return value of snprintf()... No." | tee -a configure.log - echo " WARNING: apparently snprintf() does not return a value. zlib" | tee -a configure.log - echo " can build but will be open to possible string-format security" | tee -a configure.log - echo " vulnerabilities." | tee -a configure.log - fi - else - CFLAGS="$CFLAGS -DNO_snprintf" - SFLAGS="$SFLAGS -DNO_snprintf" - echo "Checking for snprintf() in stdio.h... No." | tee -a configure.log - echo " WARNING: snprintf() not found, falling back to sprintf(). zlib" | tee -a configure.log - echo " can build but will be open to possible buffer-overflow security" | tee -a configure.log - echo " vulnerabilities." | tee -a configure.log - - echo >> configure.log - cat >$test.c < -int mytest() -{ - char buf[20]; - return sprintf(buf, "%s", "foo"); -} -int main() -{ - return (mytest()); -} -EOF - - if try $CC -c $CFLAGS $test.c; then - echo "Checking for return value of sprintf()... Yes." | tee -a configure.log - else - CFLAGS="$CFLAGS -DHAS_sprintf_void" - SFLAGS="$SFLAGS -DHAS_sprintf_void" - echo "Checking for return value of sprintf()... No." | tee -a configure.log - echo " WARNING: apparently sprintf() does not return a value. zlib" | tee -a configure.log - echo " can build but will be open to possible string-format security" | tee -a configure.log - echo " vulnerabilities." | tee -a configure.log - fi - fi -fi - -# see if we can hide zlib internal symbols that are linked between separate source files -if test "$gcc" -eq 1; then - echo >> configure.log - cat > $test.c <> configure.log -echo ALL = $ALL >> configure.log -echo AR = $AR >> configure.log -echo ARFLAGS = $ARFLAGS >> configure.log -echo CC = $CC >> configure.log -echo CFLAGS = $CFLAGS >> configure.log -echo CPP = $CPP >> configure.log -echo EXE = $EXE >> configure.log -echo LDCONFIG = $LDCONFIG >> configure.log -echo LDFLAGS = $LDFLAGS >> configure.log -echo LDSHARED = $LDSHARED >> configure.log -echo LDSHAREDLIBC = $LDSHAREDLIBC >> configure.log -echo OBJC = $OBJC >> configure.log -echo PIC_OBJC = $PIC_OBJC >> configure.log -echo RANLIB = $RANLIB >> configure.log -echo SFLAGS = $SFLAGS >> configure.log -echo SHAREDLIB = $SHAREDLIB >> configure.log -echo SHAREDLIBM = $SHAREDLIBM >> configure.log -echo SHAREDLIBV = $SHAREDLIBV >> configure.log -echo STATICLIB = $STATICLIB >> configure.log -echo TEST = $TEST >> configure.log -echo VER = $VER >> configure.log -echo Z_U4 = $Z_U4 >> configure.log -echo SRCDIR = $SRCDIR >> configure.log -echo exec_prefix = $exec_prefix >> configure.log -echo includedir = $includedir >> configure.log -echo libdir = $libdir >> configure.log -echo mandir = $mandir >> configure.log -echo prefix = $prefix >> configure.log -echo sharedlibdir = $sharedlibdir >> configure.log -echo uname = $uname >> configure.log - -# udpate Makefile with the configure results -sed < ${SRCDIR}Makefile.in " -/^CC *=/s#=.*#=$CC# -/^CFLAGS *=/s#=.*#=$CFLAGS# -/^SFLAGS *=/s#=.*#=$SFLAGS# -/^LDFLAGS *=/s#=.*#=$LDFLAGS# -/^LDSHARED *=/s#=.*#=$LDSHARED# -/^CPP *=/s#=.*#=$CPP# -/^STATICLIB *=/s#=.*#=$STATICLIB# -/^SHAREDLIB *=/s#=.*#=$SHAREDLIB# -/^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV# -/^SHAREDLIBM *=/s#=.*#=$SHAREDLIBM# -/^AR *=/s#=.*#=$AR# -/^ARFLAGS *=/s#=.*#=$ARFLAGS# -/^RANLIB *=/s#=.*#=$RANLIB# -/^LDCONFIG *=/s#=.*#=$LDCONFIG# -/^LDSHAREDLIBC *=/s#=.*#=$LDSHAREDLIBC# -/^EXE *=/s#=.*#=$EXE# -/^SRCDIR *=/s#=.*#=$SRCDIR# -/^ZINC *=/s#=.*#=$ZINC# -/^ZINCOUT *=/s#=.*#=$ZINCOUT# -/^prefix *=/s#=.*#=$prefix# -/^exec_prefix *=/s#=.*#=$exec_prefix# -/^libdir *=/s#=.*#=$libdir# -/^sharedlibdir *=/s#=.*#=$sharedlibdir# -/^includedir *=/s#=.*#=$includedir# -/^mandir *=/s#=.*#=$mandir# -/^OBJC *=/s#=.*#= $OBJC# -/^PIC_OBJC *=/s#=.*#= $PIC_OBJC# -/^all: */s#:.*#: $ALL# -/^test: */s#:.*#: $TEST# -" > Makefile - -# create zlib.pc with the configure results -sed < ${SRCDIR}zlib.pc.in " -/^CC *=/s#=.*#=$CC# -/^CFLAGS *=/s#=.*#=$CFLAGS# -/^CPP *=/s#=.*#=$CPP# -/^LDSHARED *=/s#=.*#=$LDSHARED# -/^STATICLIB *=/s#=.*#=$STATICLIB# -/^SHAREDLIB *=/s#=.*#=$SHAREDLIB# -/^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV# -/^SHAREDLIBM *=/s#=.*#=$SHAREDLIBM# -/^AR *=/s#=.*#=$AR# -/^ARFLAGS *=/s#=.*#=$ARFLAGS# -/^RANLIB *=/s#=.*#=$RANLIB# -/^EXE *=/s#=.*#=$EXE# -/^prefix *=/s#=.*#=$prefix# -/^exec_prefix *=/s#=.*#=$exec_prefix# -/^libdir *=/s#=.*#=$libdir# -/^sharedlibdir *=/s#=.*#=$sharedlibdir# -/^includedir *=/s#=.*#=$includedir# -/^mandir *=/s#=.*#=$mandir# -/^LDFLAGS *=/s#=.*#=$LDFLAGS# -" | sed -e " -s/\@VERSION\@/$VER/g; -" > zlib.pc - -# done -leave 0 diff --git a/contrib/zlib/make_vms.com b/contrib/zlib/make_vms.com deleted file mode 100644 index 65e9d0cbc..000000000 --- a/contrib/zlib/make_vms.com +++ /dev/null @@ -1,867 +0,0 @@ -$! make libz under VMS written by -$! Martin P.J. Zinser -$! -$! In case of problems with the install you might contact me at -$! zinser@zinser.no-ip.info(preferred) or -$! martin.zinser@eurexchange.com (work) -$! -$! Make procedure history for Zlib -$! -$!------------------------------------------------------------------------------ -$! Version history -$! 0.01 20060120 First version to receive a number -$! 0.02 20061008 Adapt to new Makefile.in -$! 0.03 20091224 Add support for large file check -$! 0.04 20100110 Add new gzclose, gzlib, gzread, gzwrite -$! 0.05 20100221 Exchange zlibdefs.h by zconf.h.in -$! 0.06 20120111 Fix missing amiss_err, update zconf_h.in, fix new exmples -$! subdir path, update module search in makefile.in -$! 0.07 20120115 Triggered by work done by Alexey Chupahin completly redesigned -$! shared image creation -$! 0.08 20120219 Make it work on VAX again, pre-load missing symbols to shared -$! image -$! 0.09 20120305 SMS. P1 sets builder ("MMK", "MMS", " " (built-in)). -$! "" -> automatic, preference: MMK, MMS, built-in. -$! -$ on error then goto err_exit -$! -$ true = 1 -$ false = 0 -$ tmpnam = "temp_" + f$getjpi("","pid") -$ tt = tmpnam + ".txt" -$ tc = tmpnam + ".c" -$ th = tmpnam + ".h" -$ define/nolog tconfig 'th' -$ its_decc = false -$ its_vaxc = false -$ its_gnuc = false -$ s_case = False -$! -$! Setup variables holding "config" information -$! -$ Make = "''p1'" -$ name = "Zlib" -$ version = "?.?.?" -$ v_string = "ZLIB_VERSION" -$ v_file = "zlib.h" -$ ccopt = "/include = []" -$ lopts = "" -$ dnsrl = "" -$ aconf_in_file = "zconf.h.in#zconf.h_in#zconf_h.in" -$ conf_check_string = "" -$ linkonly = false -$ optfile = name + ".opt" -$ mapfile = name + ".map" -$ libdefs = "" -$ vax = f$getsyi("HW_MODEL").lt.1024 -$ axp = f$getsyi("HW_MODEL").ge.1024 .and. f$getsyi("HW_MODEL").lt.4096 -$ ia64 = f$getsyi("HW_MODEL").ge.4096 -$! -$! 2012-03-05 SMS. -$! Why is this needed? And if it is needed, why not simply ".not. vax"? -$! -$!!! if axp .or. ia64 then set proc/parse=extended -$! -$ whoami = f$parse(f$environment("Procedure"),,,,"NO_CONCEAL") -$ mydef = F$parse(whoami,,,"DEVICE") -$ mydir = f$parse(whoami,,,"DIRECTORY") - "][" -$ myproc = f$parse(whoami,,,"Name") + f$parse(whoami,,,"type") -$! -$! Check for MMK/MMS -$! -$ if (Make .eqs. "") -$ then -$ If F$Search ("Sys$System:MMS.EXE") .nes. "" Then Make = "MMS" -$ If F$Type (MMK) .eqs. "STRING" Then Make = "MMK" -$ else -$ Make = f$edit( Make, "trim") -$ endif -$! -$ gosub find_version -$! -$ open/write topt tmp.opt -$ open/write optf 'optfile' -$! -$ gosub check_opts -$! -$! Look for the compiler used -$! -$ gosub check_compiler -$ close topt -$ close optf -$! -$ if its_decc -$ then -$ ccopt = "/prefix=all" + ccopt -$ if f$trnlnm("SYS") .eqs. "" -$ then -$ if axp -$ then -$ define sys sys$library: -$ else -$ ccopt = "/decc" + ccopt -$ define sys decc$library_include: -$ endif -$ endif -$! -$! 2012-03-05 SMS. -$! Why /NAMES = AS_IS? Why not simply ".not. vax"? And why not on VAX? -$! -$ if axp .or. ia64 -$ then -$ ccopt = ccopt + "/name=as_is/opt=(inline=speed)" -$ s_case = true -$ endif -$ endif -$ if its_vaxc .or. its_gnuc -$ then -$ if f$trnlnm("SYS").eqs."" then define sys sys$library: -$ endif -$! -$! Build a fake configure input header -$! -$ open/write conf_hin config.hin -$ write conf_hin "#undef _LARGEFILE64_SOURCE" -$ close conf_hin -$! -$! -$ i = 0 -$FIND_ACONF: -$ fname = f$element(i,"#",aconf_in_file) -$ if fname .eqs. "#" then goto AMISS_ERR -$ if f$search(fname) .eqs. "" -$ then -$ i = i + 1 -$ goto find_aconf -$ endif -$ open/read/err=aconf_err aconf_in 'fname' -$ open/write aconf zconf.h -$ACONF_LOOP: -$ read/end_of_file=aconf_exit aconf_in line -$ work = f$edit(line, "compress,trim") -$ if f$extract(0,6,work) .nes. "#undef" -$ then -$ if f$extract(0,12,work) .nes. "#cmakedefine" -$ then -$ write aconf line -$ endif -$ else -$ cdef = f$element(1," ",work) -$ gosub check_config -$ endif -$ goto aconf_loop -$ACONF_EXIT: -$ write aconf "" -$ write aconf "/* VMS specifics added by make_vms.com: */" -$ write aconf "#define VMS 1" -$ write aconf "#include " -$ write aconf "#include " -$ write aconf "#ifdef _LARGEFILE" -$ write aconf "# define off64_t __off64_t" -$ write aconf "# define fopen64 fopen" -$ write aconf "# define fseeko64 fseeko" -$ write aconf "# define lseek64 lseek" -$ write aconf "# define ftello64 ftell" -$ write aconf "#endif" -$ write aconf "#if !defined( __VAX) && (__CRTL_VER >= 70312000)" -$ write aconf "# define HAVE_VSNPRINTF" -$ write aconf "#endif" -$ close aconf_in -$ close aconf -$ if f$search("''th'") .nes. "" then delete 'th';* -$! Build the thing plain or with mms -$! -$ write sys$output "Compiling Zlib sources ..." -$ if make.eqs."" -$ then -$ if (f$search( "example.obj;*") .nes. "") then delete example.obj;* -$ if (f$search( "minigzip.obj;*") .nes. "") then delete minigzip.obj;* -$ CALL MAKE adler32.OBJ "CC ''CCOPT' adler32" - - adler32.c zlib.h zconf.h -$ CALL MAKE compress.OBJ "CC ''CCOPT' compress" - - compress.c zlib.h zconf.h -$ CALL MAKE crc32.OBJ "CC ''CCOPT' crc32" - - crc32.c zlib.h zconf.h -$ CALL MAKE deflate.OBJ "CC ''CCOPT' deflate" - - deflate.c deflate.h zutil.h zlib.h zconf.h -$ CALL MAKE gzclose.OBJ "CC ''CCOPT' gzclose" - - gzclose.c zutil.h zlib.h zconf.h -$ CALL MAKE gzlib.OBJ "CC ''CCOPT' gzlib" - - gzlib.c zutil.h zlib.h zconf.h -$ CALL MAKE gzread.OBJ "CC ''CCOPT' gzread" - - gzread.c zutil.h zlib.h zconf.h -$ CALL MAKE gzwrite.OBJ "CC ''CCOPT' gzwrite" - - gzwrite.c zutil.h zlib.h zconf.h -$ CALL MAKE infback.OBJ "CC ''CCOPT' infback" - - infback.c zutil.h inftrees.h inflate.h inffast.h inffixed.h -$ CALL MAKE inffast.OBJ "CC ''CCOPT' inffast" - - inffast.c zutil.h zlib.h zconf.h inffast.h -$ CALL MAKE inflate.OBJ "CC ''CCOPT' inflate" - - inflate.c zutil.h zlib.h zconf.h infblock.h -$ CALL MAKE inftrees.OBJ "CC ''CCOPT' inftrees" - - inftrees.c zutil.h zlib.h zconf.h inftrees.h -$ CALL MAKE trees.OBJ "CC ''CCOPT' trees" - - trees.c deflate.h zutil.h zlib.h zconf.h -$ CALL MAKE uncompr.OBJ "CC ''CCOPT' uncompr" - - uncompr.c zlib.h zconf.h -$ CALL MAKE zutil.OBJ "CC ''CCOPT' zutil" - - zutil.c zutil.h zlib.h zconf.h -$ write sys$output "Building Zlib ..." -$ CALL MAKE libz.OLB "lib/crea libz.olb *.obj" *.OBJ -$ write sys$output "Building example..." -$ CALL MAKE example.OBJ "CC ''CCOPT' [.test]example" - - [.test]example.c zlib.h zconf.h -$ call make example.exe "LINK example,libz.olb/lib" example.obj libz.olb -$ write sys$output "Building minigzip..." -$ CALL MAKE minigzip.OBJ "CC ''CCOPT' [.test]minigzip" - - [.test]minigzip.c zlib.h zconf.h -$ call make minigzip.exe - - "LINK minigzip,libz.olb/lib" - - minigzip.obj libz.olb -$ else -$ gosub crea_mms -$ write sys$output "Make ''name' ''version' with ''Make' " -$ 'make' -$ endif -$! -$! Create shareable image -$! -$ gosub crea_olist -$ write sys$output "Creating libzshr.exe" -$ call map_2_shopt 'mapfile' 'optfile' -$ LINK_'lopts'/SHARE=libzshr.exe modules.opt/opt,'optfile'/opt -$ write sys$output "Zlib build completed" -$ delete/nolog tmp.opt;* -$ exit -$AMISS_ERR: -$ write sys$output "No source for config.hin found." -$ write sys$output "Tried any of ''aconf_in_file'" -$ goto err_exit -$CC_ERR: -$ write sys$output "C compiler required to build ''name'" -$ goto err_exit -$ERR_EXIT: -$ set message/facil/ident/sever/text -$ close/nolog optf -$ close/nolog topt -$ close/nolog aconf_in -$ close/nolog aconf -$ close/nolog out -$ close/nolog min -$ close/nolog mod -$ close/nolog h_in -$ write sys$output "Exiting..." -$ exit 2 -$! -$! -$MAKE: SUBROUTINE !SUBROUTINE TO CHECK DEPENDENCIES -$ V = 'F$Verify(0) -$! P1 = What we are trying to make -$! P2 = Command to make it -$! P3 - P8 What it depends on -$ -$ If F$Search(P1) .Eqs. "" Then Goto Makeit -$ Time = F$CvTime(F$File(P1,"RDT")) -$arg=3 -$Loop: -$ Argument = P'arg -$ If Argument .Eqs. "" Then Goto Exit -$ El=0 -$Loop2: -$ File = F$Element(El," ",Argument) -$ If File .Eqs. " " Then Goto Endl -$ AFile = "" -$Loop3: -$ OFile = AFile -$ AFile = F$Search(File) -$ If AFile .Eqs. "" .Or. AFile .Eqs. OFile Then Goto NextEl -$ If F$CvTime(F$File(AFile,"RDT")) .Ges. Time Then Goto Makeit -$ Goto Loop3 -$NextEL: -$ El = El + 1 -$ Goto Loop2 -$EndL: -$ arg=arg+1 -$ If arg .Le. 8 Then Goto Loop -$ Goto Exit -$ -$Makeit: -$ VV=F$VERIFY(0) -$ write sys$output P2 -$ 'P2 -$ VV='F$Verify(VV) -$Exit: -$ If V Then Set Verify -$ENDSUBROUTINE -$!------------------------------------------------------------------------------ -$! -$! Check command line options and set symbols accordingly -$! -$!------------------------------------------------------------------------------ -$! Version history -$! 0.01 20041206 First version to receive a number -$! 0.02 20060126 Add new "HELP" target -$ CHECK_OPTS: -$ i = 1 -$ OPT_LOOP: -$ if i .lt. 9 -$ then -$ cparm = f$edit(p'i',"upcase") -$! -$! Check if parameter actually contains something -$! -$ if f$edit(cparm,"trim") .nes. "" -$ then -$ if cparm .eqs. "DEBUG" -$ then -$ ccopt = ccopt + "/noopt/deb" -$ lopts = lopts + "/deb" -$ endif -$ if f$locate("CCOPT=",cparm) .lt. f$length(cparm) -$ then -$ start = f$locate("=",cparm) + 1 -$ len = f$length(cparm) - start -$ ccopt = ccopt + f$extract(start,len,cparm) -$ if f$locate("AS_IS",f$edit(ccopt,"UPCASE")) .lt. f$length(ccopt) - - then s_case = true -$ endif -$ if cparm .eqs. "LINK" then linkonly = true -$ if f$locate("LOPTS=",cparm) .lt. f$length(cparm) -$ then -$ start = f$locate("=",cparm) + 1 -$ len = f$length(cparm) - start -$ lopts = lopts + f$extract(start,len,cparm) -$ endif -$ if f$locate("CC=",cparm) .lt. f$length(cparm) -$ then -$ start = f$locate("=",cparm) + 1 -$ len = f$length(cparm) - start -$ cc_com = f$extract(start,len,cparm) - if (cc_com .nes. "DECC") .and. - - (cc_com .nes. "VAXC") .and. - - (cc_com .nes. "GNUC") -$ then -$ write sys$output "Unsupported compiler choice ''cc_com' ignored" -$ write sys$output "Use DECC, VAXC, or GNUC instead" -$ else -$ if cc_com .eqs. "DECC" then its_decc = true -$ if cc_com .eqs. "VAXC" then its_vaxc = true -$ if cc_com .eqs. "GNUC" then its_gnuc = true -$ endif -$ endif -$ if f$locate("MAKE=",cparm) .lt. f$length(cparm) -$ then -$ start = f$locate("=",cparm) + 1 -$ len = f$length(cparm) - start -$ mmks = f$extract(start,len,cparm) -$ if (mmks .eqs. "MMK") .or. (mmks .eqs. "MMS") -$ then -$ make = mmks -$ else -$ write sys$output "Unsupported make choice ''mmks' ignored" -$ write sys$output "Use MMK or MMS instead" -$ endif -$ endif -$ if cparm .eqs. "HELP" then gosub bhelp -$ endif -$ i = i + 1 -$ goto opt_loop -$ endif -$ return -$!------------------------------------------------------------------------------ -$! -$! Look for the compiler used -$! -$! Version history -$! 0.01 20040223 First version to receive a number -$! 0.02 20040229 Save/set value of decc$no_rooted_search_lists -$! 0.03 20060202 Extend handling of GNU C -$! 0.04 20090402 Compaq -> hp -$CHECK_COMPILER: -$ if (.not. (its_decc .or. its_vaxc .or. its_gnuc)) -$ then -$ its_decc = (f$search("SYS$SYSTEM:DECC$COMPILER.EXE") .nes. "") -$ its_vaxc = .not. its_decc .and. (F$Search("SYS$System:VAXC.Exe") .nes. "") -$ its_gnuc = .not. (its_decc .or. its_vaxc) .and. (f$trnlnm("gnu_cc") .nes. "") -$ endif -$! -$! Exit if no compiler available -$! -$ if (.not. (its_decc .or. its_vaxc .or. its_gnuc)) -$ then goto CC_ERR -$ else -$ if its_decc -$ then -$ write sys$output "CC compiler check ... hp C" -$ if f$trnlnm("decc$no_rooted_search_lists") .nes. "" -$ then -$ dnrsl = f$trnlnm("decc$no_rooted_search_lists") -$ endif -$ define/nolog decc$no_rooted_search_lists 1 -$ else -$ if its_vaxc then write sys$output "CC compiler check ... VAX C" -$ if its_gnuc -$ then -$ write sys$output "CC compiler check ... GNU C" -$ if f$trnlnm(topt) then write topt "gnu_cc:[000000]gcclib.olb/lib" -$ if f$trnlnm(optf) then write optf "gnu_cc:[000000]gcclib.olb/lib" -$ cc = "gcc" -$ endif -$ if f$trnlnm(topt) then write topt "sys$share:vaxcrtl.exe/share" -$ if f$trnlnm(optf) then write optf "sys$share:vaxcrtl.exe/share" -$ endif -$ endif -$ return -$!------------------------------------------------------------------------------ -$! -$! If MMS/MMK are available dump out the descrip.mms if required -$! -$CREA_MMS: -$ write sys$output "Creating descrip.mms..." -$ create descrip.mms -$ open/append out descrip.mms -$ copy sys$input: out -$ deck -# descrip.mms: MMS description file for building zlib on VMS -# written by Martin P.J. Zinser -# - -OBJS = adler32.obj, compress.obj, crc32.obj, gzclose.obj, gzlib.obj\ - gzread.obj, gzwrite.obj, uncompr.obj, infback.obj\ - deflate.obj, trees.obj, zutil.obj, inflate.obj, \ - inftrees.obj, inffast.obj - -$ eod -$ write out "CFLAGS=", ccopt -$ write out "LOPTS=", lopts -$ write out "all : example.exe minigzip.exe libz.olb" -$ copy sys$input: out -$ deck - @ write sys$output " Example applications available" - -libz.olb : libz.olb($(OBJS)) - @ write sys$output " libz available" - -example.exe : example.obj libz.olb - link $(LOPTS) example,libz.olb/lib - -minigzip.exe : minigzip.obj libz.olb - link $(LOPTS) minigzip,libz.olb/lib - -clean : - delete *.obj;*,libz.olb;*,*.opt;*,*.exe;* - - -# Other dependencies. -adler32.obj : adler32.c zutil.h zlib.h zconf.h -compress.obj : compress.c zlib.h zconf.h -crc32.obj : crc32.c zutil.h zlib.h zconf.h -deflate.obj : deflate.c deflate.h zutil.h zlib.h zconf.h -example.obj : [.test]example.c zlib.h zconf.h -gzclose.obj : gzclose.c zutil.h zlib.h zconf.h -gzlib.obj : gzlib.c zutil.h zlib.h zconf.h -gzread.obj : gzread.c zutil.h zlib.h zconf.h -gzwrite.obj : gzwrite.c zutil.h zlib.h zconf.h -inffast.obj : inffast.c zutil.h zlib.h zconf.h inftrees.h inffast.h -inflate.obj : inflate.c zutil.h zlib.h zconf.h -inftrees.obj : inftrees.c zutil.h zlib.h zconf.h inftrees.h -minigzip.obj : [.test]minigzip.c zlib.h zconf.h -trees.obj : trees.c deflate.h zutil.h zlib.h zconf.h -uncompr.obj : uncompr.c zlib.h zconf.h -zutil.obj : zutil.c zutil.h zlib.h zconf.h -infback.obj : infback.c zutil.h inftrees.h inflate.h inffast.h inffixed.h -$ eod -$ close out -$ return -$!------------------------------------------------------------------------------ -$! -$! Read list of core library sources from makefile.in and create options -$! needed to build shareable image -$! -$CREA_OLIST: -$ open/read min makefile.in -$ open/write mod modules.opt -$ src_check_list = "OBJZ =#OBJG =" -$MRLOOP: -$ read/end=mrdone min rec -$ i = 0 -$SRC_CHECK_LOOP: -$ src_check = f$element(i, "#", src_check_list) -$ i = i+1 -$ if src_check .eqs. "#" then goto mrloop -$ if (f$extract(0,6,rec) .nes. src_check) then goto src_check_loop -$ rec = rec - src_check -$ gosub extra_filnam -$ if (f$element(1,"\",rec) .eqs. "\") then goto mrloop -$MRSLOOP: -$ read/end=mrdone min rec -$ gosub extra_filnam -$ if (f$element(1,"\",rec) .nes. "\") then goto mrsloop -$MRDONE: -$ close min -$ close mod -$ return -$!------------------------------------------------------------------------------ -$! -$! Take record extracted in crea_olist and split it into single filenames -$! -$EXTRA_FILNAM: -$ myrec = f$edit(rec - "\", "trim,compress") -$ i = 0 -$FELOOP: -$ srcfil = f$element(i," ", myrec) -$ if (srcfil .nes. " ") -$ then -$ write mod f$parse(srcfil,,,"NAME"), ".obj" -$ i = i + 1 -$ goto feloop -$ endif -$ return -$!------------------------------------------------------------------------------ -$! -$! Find current Zlib version number -$! -$FIND_VERSION: -$ open/read h_in 'v_file' -$hloop: -$ read/end=hdone h_in rec -$ rec = f$edit(rec,"TRIM") -$ if (f$extract(0,1,rec) .nes. "#") then goto hloop -$ rec = f$edit(rec - "#", "TRIM") -$ if f$element(0," ",rec) .nes. "define" then goto hloop -$ if f$element(1," ",rec) .eqs. v_string -$ then -$ version = 'f$element(2," ",rec)' -$ goto hdone -$ endif -$ goto hloop -$hdone: -$ close h_in -$ return -$!------------------------------------------------------------------------------ -$! -$CHECK_CONFIG: -$! -$ in_ldef = f$locate(cdef,libdefs) -$ if (in_ldef .lt. f$length(libdefs)) -$ then -$ write aconf "#define ''cdef' 1" -$ libdefs = f$extract(0,in_ldef,libdefs) + - - f$extract(in_ldef + f$length(cdef) + 1, - - f$length(libdefs) - in_ldef - f$length(cdef) - 1, - - libdefs) -$ else -$ if (f$type('cdef') .eqs. "INTEGER") -$ then -$ write aconf "#define ''cdef' ", 'cdef' -$ else -$ if (f$type('cdef') .eqs. "STRING") -$ then -$ write aconf "#define ''cdef' ", """", '''cdef'', """" -$ else -$ gosub check_cc_def -$ endif -$ endif -$ endif -$ return -$!------------------------------------------------------------------------------ -$! -$! Check if this is a define relating to the properties of the C/C++ -$! compiler -$! -$ CHECK_CC_DEF: -$ if (cdef .eqs. "_LARGEFILE64_SOURCE") -$ then -$ copy sys$input: 'tc' -$ deck -#include "tconfig" -#define _LARGEFILE -#include - -int main(){ -FILE *fp; - fp = fopen("temp.txt","r"); - fseeko(fp,1,SEEK_SET); - fclose(fp); -} - -$ eod -$ test_inv = false -$ comm_h = false -$ gosub cc_prop_check -$ return -$ endif -$ write aconf "/* ", line, " */" -$ return -$!------------------------------------------------------------------------------ -$! -$! Check for properties of C/C++ compiler -$! -$! Version history -$! 0.01 20031020 First version to receive a number -$! 0.02 20031022 Added logic for defines with value -$! 0.03 20040309 Make sure local config file gets not deleted -$! 0.04 20041230 Also write include for configure run -$! 0.05 20050103 Add processing of "comment defines" -$CC_PROP_CHECK: -$ cc_prop = true -$ is_need = false -$ is_need = (f$extract(0,4,cdef) .eqs. "NEED") .or. (test_inv .eq. true) -$ if f$search(th) .eqs. "" then create 'th' -$ set message/nofac/noident/nosever/notext -$ on error then continue -$ cc 'tmpnam' -$ if .not. ($status) then cc_prop = false -$ on error then continue -$! The headers might lie about the capabilities of the RTL -$ link 'tmpnam',tmp.opt/opt -$ if .not. ($status) then cc_prop = false -$ set message/fac/ident/sever/text -$ on error then goto err_exit -$ delete/nolog 'tmpnam'.*;*/exclude='th' -$ if (cc_prop .and. .not. is_need) .or. - - (.not. cc_prop .and. is_need) -$ then -$ write sys$output "Checking for ''cdef'... yes" -$ if f$type('cdef_val'_yes) .nes. "" -$ then -$ if f$type('cdef_val'_yes) .eqs. "INTEGER" - - then call write_config f$fao("#define !AS !UL",cdef,'cdef_val'_yes) -$ if f$type('cdef_val'_yes) .eqs. "STRING" - - then call write_config f$fao("#define !AS !AS",cdef,'cdef_val'_yes) -$ else -$ call write_config f$fao("#define !AS 1",cdef) -$ endif -$ if (cdef .eqs. "HAVE_FSEEKO") .or. (cdef .eqs. "_LARGE_FILES") .or. - - (cdef .eqs. "_LARGEFILE64_SOURCE") then - - call write_config f$string("#define _LARGEFILE 1") -$ else -$ write sys$output "Checking for ''cdef'... no" -$ if (comm_h) -$ then - call write_config f$fao("/* !AS */",line) -$ else -$ if f$type('cdef_val'_no) .nes. "" -$ then -$ if f$type('cdef_val'_no) .eqs. "INTEGER" - - then call write_config f$fao("#define !AS !UL",cdef,'cdef_val'_no) -$ if f$type('cdef_val'_no) .eqs. "STRING" - - then call write_config f$fao("#define !AS !AS",cdef,'cdef_val'_no) -$ else -$ call write_config f$fao("#undef !AS",cdef) -$ endif -$ endif -$ endif -$ return -$!------------------------------------------------------------------------------ -$! -$! Check for properties of C/C++ compiler with multiple result values -$! -$! Version history -$! 0.01 20040127 First version -$! 0.02 20050103 Reconcile changes from cc_prop up to version 0.05 -$CC_MPROP_CHECK: -$ cc_prop = true -$ i = 1 -$ idel = 1 -$ MT_LOOP: -$ if f$type(result_'i') .eqs. "STRING" -$ then -$ set message/nofac/noident/nosever/notext -$ on error then continue -$ cc 'tmpnam'_'i' -$ if .not. ($status) then cc_prop = false -$ on error then continue -$! The headers might lie about the capabilities of the RTL -$ link 'tmpnam'_'i',tmp.opt/opt -$ if .not. ($status) then cc_prop = false -$ set message/fac/ident/sever/text -$ on error then goto err_exit -$ delete/nolog 'tmpnam'_'i'.*;* -$ if (cc_prop) -$ then -$ write sys$output "Checking for ''cdef'... ", mdef_'i' -$ if f$type(mdef_'i') .eqs. "INTEGER" - - then call write_config f$fao("#define !AS !UL",cdef,mdef_'i') -$ if f$type('cdef_val'_yes) .eqs. "STRING" - - then call write_config f$fao("#define !AS !AS",cdef,mdef_'i') -$ goto msym_clean -$ else -$ i = i + 1 -$ goto mt_loop -$ endif -$ endif -$ write sys$output "Checking for ''cdef'... no" -$ call write_config f$fao("#undef !AS",cdef) -$ MSYM_CLEAN: -$ if (idel .le. msym_max) -$ then -$ delete/sym mdef_'idel' -$ idel = idel + 1 -$ goto msym_clean -$ endif -$ return -$!------------------------------------------------------------------------------ -$! -$! Write configuration to both permanent and temporary config file -$! -$! Version history -$! 0.01 20031029 First version to receive a number -$! -$WRITE_CONFIG: SUBROUTINE -$ write aconf 'p1' -$ open/append confh 'th' -$ write confh 'p1' -$ close confh -$ENDSUBROUTINE -$!------------------------------------------------------------------------------ -$! -$! Analyze the project map file and create the symbol vector for a shareable -$! image from it -$! -$! Version history -$! 0.01 20120128 First version -$! 0.02 20120226 Add pre-load logic -$! -$ MAP_2_SHOPT: Subroutine -$! -$ SAY := "WRITE_ SYS$OUTPUT" -$! -$ IF F$SEARCH("''P1'") .EQS. "" -$ THEN -$ SAY "MAP_2_SHOPT-E-NOSUCHFILE: Error, inputfile ''p1' not available" -$ goto exit_m2s -$ ENDIF -$ IF "''P2'" .EQS. "" -$ THEN -$ SAY "MAP_2_SHOPT: Error, no output file provided" -$ goto exit_m2s -$ ENDIF -$! -$ module1 = "deflate#deflateEnd#deflateInit_#deflateParams#deflateSetDictionary" -$ module2 = "gzclose#gzerror#gzgetc#gzgets#gzopen#gzprintf#gzputc#gzputs#gzread" -$ module3 = "gzseek#gztell#inflate#inflateEnd#inflateInit_#inflateSetDictionary" -$ module4 = "inflateSync#uncompress#zlibVersion#compress" -$ open/read map 'p1 -$ if axp .or. ia64 -$ then -$ open/write aopt a.opt -$ open/write bopt b.opt -$ write aopt " CASE_SENSITIVE=YES" -$ write bopt "SYMBOL_VECTOR= (-" -$ mod_sym_num = 1 -$ MOD_SYM_LOOP: -$ if f$type(module'mod_sym_num') .nes. "" -$ then -$ mod_in = 0 -$ MOD_SYM_IN: -$ shared_proc = f$element(mod_in, "#", module'mod_sym_num') -$ if shared_proc .nes. "#" -$ then -$ write aopt f$fao(" symbol_vector=(!AS/!AS=PROCEDURE)",- - f$edit(shared_proc,"upcase"),shared_proc) -$ write bopt f$fao("!AS=PROCEDURE,-",shared_proc) -$ mod_in = mod_in + 1 -$ goto mod_sym_in -$ endif -$ mod_sym_num = mod_sym_num + 1 -$ goto mod_sym_loop -$ endif -$MAP_LOOP: -$ read/end=map_end map line -$ if (f$locate("{",line).lt. f$length(line)) .or. - - (f$locate("global:", line) .lt. f$length(line)) -$ then -$ proc = true -$ goto map_loop -$ endif -$ if f$locate("}",line).lt. f$length(line) then proc = false -$ if f$locate("local:", line) .lt. f$length(line) then proc = false -$ if proc -$ then -$ shared_proc = f$edit(line,"collapse") -$ chop_semi = f$locate(";", shared_proc) -$ if chop_semi .lt. f$length(shared_proc) then - - shared_proc = f$extract(0, chop_semi, shared_proc) -$ write aopt f$fao(" symbol_vector=(!AS/!AS=PROCEDURE)",- - f$edit(shared_proc,"upcase"),shared_proc) -$ write bopt f$fao("!AS=PROCEDURE,-",shared_proc) -$ endif -$ goto map_loop -$MAP_END: -$ close/nolog aopt -$ close/nolog bopt -$ open/append libopt 'p2' -$ open/read aopt a.opt -$ open/read bopt b.opt -$ALOOP: -$ read/end=aloop_end aopt line -$ write libopt line -$ goto aloop -$ALOOP_END: -$ close/nolog aopt -$ sv = "" -$BLOOP: -$ read/end=bloop_end bopt svn -$ if (svn.nes."") -$ then -$ if (sv.nes."") then write libopt sv -$ sv = svn -$ endif -$ goto bloop -$BLOOP_END: -$ write libopt f$extract(0,f$length(sv)-2,sv), "-" -$ write libopt ")" -$ close/nolog bopt -$ delete/nolog/noconf a.opt;*,b.opt;* -$ else -$ if vax -$ then -$ open/append libopt 'p2' -$ mod_sym_num = 1 -$ VMOD_SYM_LOOP: -$ if f$type(module'mod_sym_num') .nes. "" -$ then -$ mod_in = 0 -$ VMOD_SYM_IN: -$ shared_proc = f$element(mod_in, "#", module'mod_sym_num') -$ if shared_proc .nes. "#" -$ then -$ write libopt f$fao("UNIVERSAL=!AS",- - f$edit(shared_proc,"upcase")) -$ mod_in = mod_in + 1 -$ goto vmod_sym_in -$ endif -$ mod_sym_num = mod_sym_num + 1 -$ goto vmod_sym_loop -$ endif -$VMAP_LOOP: -$ read/end=vmap_end map line -$ if (f$locate("{",line).lt. f$length(line)) .or. - - (f$locate("global:", line) .lt. f$length(line)) -$ then -$ proc = true -$ goto vmap_loop -$ endif -$ if f$locate("}",line).lt. f$length(line) then proc = false -$ if f$locate("local:", line) .lt. f$length(line) then proc = false -$ if proc -$ then -$ shared_proc = f$edit(line,"collapse") -$ chop_semi = f$locate(";", shared_proc) -$ if chop_semi .lt. f$length(shared_proc) then - - shared_proc = f$extract(0, chop_semi, shared_proc) -$ write libopt f$fao("UNIVERSAL=!AS",- - f$edit(shared_proc,"upcase")) -$ endif -$ goto vmap_loop -$VMAP_END: -$ else -$ write sys$output "Unknown Architecture (Not VAX, AXP, or IA64)" -$ write sys$output "No options file created" -$ endif -$ endif -$ EXIT_M2S: -$ close/nolog map -$ close/nolog libopt -$ endsubroutine diff --git a/contrib/zlib/treebuild.xml b/contrib/zlib/treebuild.xml deleted file mode 100644 index 6bd6677c4..000000000 --- a/contrib/zlib/treebuild.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - zip compression library - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/zlib/zlib.3 b/contrib/zlib/zlib.3 deleted file mode 100644 index fac206b52..000000000 --- a/contrib/zlib/zlib.3 +++ /dev/null @@ -1,149 +0,0 @@ -.TH ZLIB 3 "xx Jan 2017" -.SH NAME -zlib \- compression/decompression library -.SH SYNOPSIS -[see -.I zlib.h -for full description] -.SH DESCRIPTION -The -.I zlib -library is a general purpose data compression library. -The code is thread safe, assuming that the standard library functions -used are thread safe, such as memory allocation routines. -It provides in-memory compression and decompression functions, -including integrity checks of the uncompressed data. -This version of the library supports only one compression method (deflation) -but other algorithms may be added later -with the same stream interface. -.LP -Compression can be done in a single step if the buffers are large enough -or can be done by repeated calls of the compression function. -In the latter case, -the application must provide more input and/or consume the output -(providing more output space) before each call. -.LP -The library also supports reading and writing files in -.IR gzip (1) -(.gz) format -with an interface similar to that of stdio. -.LP -The library does not install any signal handler. -The decoder checks the consistency of the compressed data, -so the library should never crash even in the case of corrupted input. -.LP -All functions of the compression library are documented in the file -.IR zlib.h . -The distribution source includes examples of use of the library -in the files -.I test/example.c -and -.IR test/minigzip.c, -as well as other examples in the -.IR examples/ -directory. -.LP -Changes to this version are documented in the file -.I ChangeLog -that accompanies the source. -.LP -.I zlib -is built in to many languages and operating systems, including but not limited to -Java, Python, .NET, PHP, Perl, Ruby, Swift, and Go. -.LP -An experimental package to read and write files in the .zip format, -written on top of -.I zlib -by Gilles Vollant (info@winimage.com), -is available at: -.IP -http://www.winimage.com/zLibDll/minizip.html -and also in the -.I contrib/minizip -directory of the main -.I zlib -source distribution. -.SH "SEE ALSO" -The -.I zlib -web site can be found at: -.IP -http://zlib.net/ -.LP -The data format used by the -.I zlib -library is described by RFC -(Request for Comments) 1950 to 1952 in the files: -.IP -http://tools.ietf.org/html/rfc1950 (for the zlib header and trailer format) -.br -http://tools.ietf.org/html/rfc1951 (for the deflate compressed data format) -.br -http://tools.ietf.org/html/rfc1952 (for the gzip header and trailer format) -.LP -Mark Nelson wrote an article about -.I zlib -for the Jan. 1997 issue of Dr. Dobb's Journal; -a copy of the article is available at: -.IP -http://marknelson.us/1997/01/01/zlib-engine/ -.SH "REPORTING PROBLEMS" -Before reporting a problem, -please check the -.I zlib -web site to verify that you have the latest version of -.IR zlib ; -otherwise, -obtain the latest version and see if the problem still exists. -Please read the -.I zlib -FAQ at: -.IP -http://zlib.net/zlib_faq.html -.LP -before asking for help. -Send questions and/or comments to zlib@gzip.org, -or (for the Windows DLL version) to Gilles Vollant (info@winimage.com). -.SH AUTHORS AND LICENSE -Version 1.2.11.1 -.LP -Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler -.LP -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. -.LP -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: -.LP -.nr step 1 1 -.IP \n[step]. 3 -The origin of this software must not be misrepresented; you must not -claim that you wrote the original software. If you use this software -in a product, an acknowledgment in the product documentation would be -appreciated but is not required. -.IP \n+[step]. -Altered source versions must be plainly marked as such, and must not be -misrepresented as being the original software. -.IP \n+[step]. -This notice may not be removed or altered from any source distribution. -.LP -Jean-loup Gailly Mark Adler -.br -jloup@gzip.org madler@alumni.caltech.edu -.LP -The deflate format used by -.I zlib -was defined by Phil Katz. -The deflate and -.I zlib -specifications were written by L. Peter Deutsch. -Thanks to all the people who reported problems and suggested various -improvements in -.IR zlib ; -who are too numerous to cite here. -.LP -UNIX manual page by R. P. C. Rodgers, -U.S. National Library of Medicine (rodgers@nlm.nih.gov). -.\" end of man page diff --git a/contrib/zlib/zlib.3.pdf b/contrib/zlib/zlib.3.pdf deleted file mode 100644 index 48e6147385dc9dfc0058474f2e1efa1453409e8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19320 zcmch<1yo$ivH%)f1Hm<9u;2`XJ0Z9QcbCDP!2^WgmIQZq3GNbt1$Wn=!6lHO0scU8 z&dELJ-uLeR>#aYtW~O&{S9NtsckSxhL#-ex&IDp+N2TuGdz*#I0bl{x8Cjz8^8;nU zwq_7>04KCY2`Fx14FNj>fa2DM5U?oN*v@ zq<8QqT8NH6&obFpvhGp>TTM&b_)<|B$-%P2XvMV}o++O`Is3N2@`_fqQu{?L7$#wR zThQwgA}?6=4h?_o$;p|cr{gKPvFoB#&%q19bmZ^v5?kZ6PmhU#>)*n5=W(th3=&N= zNvn4ErBv4W+vLw^+CBr9nwwFwl8Kyz21{>4$`Ianhxw}oE4g?%U^jby?Z4RLvjhL| zW~OS=Ys1jmo6plLzbywhE$$bW7SDgNR&~EMa{G|KA*OZ2?x%VVVRla`9_|__XdD;{@jMf`9_^1tBRXNo`T%hBqex z3T6i*>W(g^*^g3}v=mkU*I~^^7+04+b|!9ZYoZ)6zZURj+HOmVAurrh9sb^bf3PK{whk~ zIm~0db)Mm7G)-okPj^vLDup?Qi=1gE|=v>NI&R;nPvsj@TG1|!`<3V*^d{|^jQ z@-2rH*SgMqeTLgHG#E_6(~oPNo$~QRLKDmq{RDtx&%Kt)`pHPtN_P8ARS(oLcE$YT zSueu%!wMcv5jQREXKG?g^SW#m;W3;|DtBW=u1lF@(bcn~E%T@Pq!}lZ##yPkSZmv8 zgDpizKkizzNAp*8`_OvDOOPRl$}&~WR`*ARgRf3UrsZ-u1JP7pfKi5VNLsi$l%j$} zG(4SiH1Kso+gCj5)ZHbT=9sFe6g|BQBUMUlzodPY%6Sju&RQ?g$IvcP5C86qBql?j|x5ZW)){f%?kV-7D9?*f>_2j5?%z0L1w2Xh2E%_G8?x zB@Ye)@R$li(7B|A+T0>9bnIQX{p+t-u%%z&{UWYkq2-dQFN?fJK{N4C->PoWqZIlu zU=l81f~Df=Y}xWQKAk)Re6fcNabm^+0ryjd<9*b$95S(e(&yAxX*~lQBUM@mBRd5nK{A&&SD_Uoq1X^^ zL`FOSAEYgY0$5%^bD>Zkgui<6wL5R&vucUuT*qP(`Li26w8!$jV&=-nZ8=ma2twTL zGX91)UNyzb@9?H(tr^83rH_MFI0&58v=b>}L$sE>$d>trUk zhT3$V!dBnbpO!OV*OyaJ^Srmv-*O4&C$DNu_3fXq`c_ZhR=rO15HZ3RjaQ;*dv7jp z(UnHA^Mu8{6a$m$3;ua<(R__;{*pCWke0=_$Jt53rr6z!Zk$UYt z*={E{-|+(XrDSvEXe?~LntiN}X{1cm$tn6>Sd{~wHBphvSnFuzT~7d3N}qe z+UX?b7PW&1es?BpZB|ZYSGFWP9%gE+oVuS#(uQArfm1Z98xKlVMa&rkN{lEUE}|HF zfuB4>QVe|H!}rW6yQk%v6?v?EVnvpO68y67laN2CeE1|zj44ey1AM=(IGhpLVwNj` zJ)6;5K}T}98M0)?c}2+1Aq{JyNroVI7z+=xXpT_n$n%xp!fEzE&J=5$)TJ{tR0jR5 zpIc;*to{k!f@D6yt0BPmH}(Zi?yuj_c_}f#zPavXDE$*T74Q6FS2f32^!xYU%%H#E-zpEUL5C>E6>1A~myyQiJjiKe{} zllmfhLbOSwkRJCzc7}>$mglp`Sr>8J~zPzfNx2;)ZCguEVe}z#;mB8>HFGshOP+U!4=wN*E7R%9;qf??cf3TUEIuHM@ z84(h|7o>e*e1zhb`LS>|0QkI@MRZbHA(oW%2O>=2IdSTPz>r+T0jW! zR&@0P#`ORv8`UH2UAU)d;?7lcB$h%u;%9iG2K#T%3XrwyT>&@>gM)i9ruw=4IBZmX z?t3w6+ccqNFEh#rY>!^mR_(QNjYO^;;2q~5x_(YiUHtrc7+aa_+~8dZkO*?RpQYEl zWUT^|eVLn`Oo?QZPmJHFys_)uqzF)445wpTBg7j)A6g~fQK%PyZMAO-N5=Wi$v37P z(J#$d*ib`P(gsuctRgYI0i|%h(l&2PxF<+faAn8cScuB|BrHCY`-)sNQqTcQb@dgaX;y;-$$ve7v4A*JcqR>6kf#>uFS14pg zd~d~1U$Q)A*V@|yq3A-vOYoTsD_hSqAj)uyWt|EFZ}ml%8sg?Ol zrHM)@Ol=j&$wd;?(!Qa5{@HVIkiq!z51|CQH;GbCgWg+*vYs{({gb&1RiC!iZ|v6A zL2Ctd3Fiv_IA@u`qQhE*qq(x|{%(LR{Rb=Z^!wOpqwhYe3{$Z&EjFR1e11k&ZLcWR z_mWc`_Nnaa4|+-*TGc@}qXqO?-=n%!@CAxyDJt8z2s{8qqNEL@BudS%Op(K8b}v3# zdW(;%a*}F3xYlxA^d)}5#wLNt=kU!#Xf9~Ix(Ur)z(|%$QewVFL0MLMt;_Xkz&>n4 z3o1%uU3cplh0{z`-og(SR`TcM!~RwR5_OaT$rDKXwwCqzN>Ao!J~RrrL>Svk6IOw8 zkMoQ#HHvbcFJ|4KyoUv?Tg4(*o)f~#Wm%&HN5GN@LCkR5suV_i;M3eE7bfMvb7P8= zgX`I+#~l|NSD203Qm=Qp4!up|57H0y^-^vfIV=WRqJC61a>Q#Vy5`1dN#|F5{Xm@8 zfaM$wuFIKj!l?m5c0DzeXz!(&eWq*_`^FB8%F48tkIvU^lXc zEp!F55u~@uVG*QT($7@xam19g#=NovR+3cHduHsi0 zmDgD#s}xxE1uPC6BR)NYd%dogTG_hqn1cJs>0Tiv--mRqUxBB_5`y^Nei!=DS>NFPiu#;giE^(PuH*Vm@)|>9@|I zzQKl%C2z|ju8>-lE(Z7wm&to~rQgCjMtSytZB2gN9YRZYca(Sck$1mroGk39zv@6- z|G3=*asM~>o6Z{ZQ4`$QUS(w?Brc-yQ&BG$-7|y8T?{=m@Ixaa0=(>-CMFT}QT1s< z?Gmmx1?V(ui(hR8vDU`)@w*?Jgbp(1Gz{isw27tccTZ_q>R%~f{<^U-+V6v+C>Y_RX z8z2W7?rx;!0kZYdsW!VuNQXqL1e*9ZU;y^1(&k{TWjE!Oww~H#;>KXwb)>5)pS&z7 z1&sgb`bz$M*Y|fp2j4%8nGUNJ#RNHiRWm5|y=|%UWb~Y>!|dqoo?=u^jCW01*{HDc z${;~r@uEBP7*1QiuH8#tC|Bm;uU&gk_v7|J&-Qs~467OC`$W|I@nMgBl^D9Kw4vIJ zmjW)0hni^CJI1Q3{y6(Ruw+ygrs}OhT&ZurAqmHMxU;1UMxrPgzEL@cUCyJtKw~XG zhx>uEfzA=}guV+*?D?Kk?IQ}2oZ2cNE;<&-&<$5Ax-7^J_q>drlRpVhJNBtgXb*_C zJopRc$hL<|*=%;{X$k#_r@j(=$T9sO*~>KuG8Zq>N{VU-P5g%w@2?`{PbC;g6*rL< z*;tHZq-3&37_KUL%7aSZ_X=>HCe>iN=~R*^8f^zPEz9j`XBv?`I1K17V|swYz3fp* zALyY=J&TT^CKEkb$+hv?2~&9yMT<00X9&wpS5R9D%Oft3tuv!-1e@8VDut8gYs!X` zXJqT<$ z^eHEPpXCT`MO%Sn{ux+Ewwau?&r@IaUASK!7&CQh@**lu@wG~m$Q8$rQPi8SH`&aH zywSAvtL~~bA1HT1vLse8t&Z`Zj^@t~NBeY&N;YR_j#=hBlaAx{01}SaQ)pacy(k~G0#fFx?q(!8S@d#pMbz9O5$#%wC^YOB zIKOqOt!z%W3`wXSv3Y)_r;MfKqE*x$q7~=OFqt#fyhkMDucoLOCyqBFy?XvUfd5<_ zc8piQU!%Z{rLN&ix6M$MuXCfVE|BG#R@RMFw|Yl2Z)%bFy!L~v(hdh&E!I&=gXM)m za-I$MkdiDbAD^Qh9oc*P08s={h(i>yrV+2yLn?06Q`s4*cg)&v!>5hYh`e#1(wg~T zA))*1MLV#pwj=oP4ZA9MS06Rly`;*P$NVfk-rbIlQz;VQX2XYN*92GjoHOmqqy`r? zus{{}tft<>ZSf|h4gww+pg&%6ur zEMUZU07;(wE$v&td_#eb2}#erX**T-!@#hurb03%GqvoO#K>!%5B+7%_S8@CW|*V9 z8alc2Z>>{Yt58_@!=l}l+*%So;^syr61@d?c@GEKEqs1jPG)A+qtF$IQ~Us-QI{&a z_<;|;x{p8EO+ymqLPAXd8tpfKwH2mNle z?L74-k5%yr3Xjfr!*)=kMhvqp_Ri(rieO4d5`Y955}7fOT@iYd-qzadhhZC?=bhKbL#}5L!C?#z<@=Uq**W-T; zOE%#Y?`!b>s<9Ob(r?3m3g=zQfypMqT<+cPn9HhZJSyLRn(nfL&Pd}mr@zS8s{ z**x!Y1BPgrV}?~vZg6pUStu_@G(7d{Vllz3c#D?E1eSQUA7v>q&f~48kC6Fn+B^9n zDWA`uQ_VzUJ@(H!Nb>lR2sANx@|lrJ%D6OYMAflLD5EP0WWBg@@bx$~?gbrOjN}p)dsTDSIA6L3?R5vvP|MnhDz&FOa@$ zXIbZ60Vegh2|j!3?D3H-kLMM+D+?ZXupb~w5g?CK&8OMjt7s^7wJrA7#Kp6`YCxABU z&l!X6gZrqu0O(Ymp>t=woAw`b{1d}31R{nILu)&;KQR4bvm6*=Xaa2ku>PG%zxj6e zJo=O0s6crIIRHpd@Q;yx6Soo<>K&ZYZ*=^2B%uUiMTN>8+746#v7mzPIRRASM!k0$ ze}lQ>*U#w#|B3OQfq%6Aw;1n6{RQ*f&UWNqQ# z48B(+QLvM-qlG=h&Jh5*=haV5xdVi1nuVhiM8w?C5x~X@lr{WS2C=dN)h$dQ=1$tE ztXy|<`giZ2EqCA059+_`@mt-mb}rUC`v3LK!Eq;j==;CGao6|$!*U1l6Y`Hb)?c{q ze!1D$e)i+wME%)|?XCt2_1ApvuAn$rSnh1l7J8CoVQU7Rwym(Olf|Fc%FafRJIlY* z9H^%Jsg!r1e;0$6@^dK-jo-xZ+|JtMA2I+zA(j4j8F2Ib7b3W$0{Z?h7S_Y3{p>&PE^^p{rsu3^7={+j%q%|jRH zd#*q^^9N`DL5&E4?#SQga>Vp;1Jg{0U5{j zAXr#f7{4q4A^c-J-1cQ0m36#F3OXqW>hB%iN4|ewR`{vNM#0LZ_!i=IR(zG}bWrHz zFfak%;jl0!Mlb)a$YjNk!9`}ha>i|YU|VU9IG-SScM3v1>) zua3Evde?_ZYn+1`o6jhY?W3?xlE+Q!NN^!Zxwc{GK%uUhpo4$o!-3v`PmjS(zlKrN zj!k{A(mU2eo5=skbb}r%aQ{sG)CV4(eB`fi zpLuBw6KJf)=stjp);VWS(){RYNw-QRgt&v_1L*VSFX9D&e zW||n1K)w%gJBVr}@>(D2l6;`4|E}@Y{v4_LiDg(({Pvk;i?8%zA4xNj(Pr$b3qX?A z`SdLWvCl$(cgyd%)Y7j!bJKfGIfF5eLM8iKkYdVyXCi%`Z~QgOwWU!@j-5pILl&py zt9MKcob&K(pJ9zr_#g)nhIxBk4ea>X8~RX4?gqS zYOiZ8+()&TUnf^OYHkx176OW=rA7SO9`ae2q z&Pq_hc&fF{*7t^hEVtecjrADQjPZ462dKBu0VT6A^V$A8HIn1WfE|hztz&E)Zm^pAT=z2y8UDMGNCCpo;|-0e$IDm zfK(E}`$weFd-2r#A%Zs}q(-k(h=L>a*uq^SvLmNS8*3usBcl;CpLONo?XcTRzsC|3 zc0DeJ$q4?uw(x^M++?@Q9vi{f#86Jy!eYDOgiG2EcG5wHC^FVFyelC!hFprj@tQDQ zGX_77j1e>Z4ax&M%kjrwMc0NAs3TrC8rh6~N8Cx!jMLN@Y>Q(f?kU4g@&6HhVHw)? zjtw|ztoxBSI}8A9E@vZZiYVds}*AffxZkZBjXk$C!_W#X8TZszSbM z2mJ_+rMe8HGh!ur{1u38BSBu4_-J*JanR7`@jg#@ZRJ$I>H9h)seB{1fg*WOCtKx9 z|7A`-y$_j$)dzt$b6?^PZk~#^Qw7NGcgIPe=2CYdM~y3wua9CXL`+(VcaxZ-{z!JM zY62`8kXs)ishY0_Lup76rj7L8>-Z7R z?WxqTnmLbsfxfu#ij(=xo=e7R=XI%#cQ*tplR@umQlANYz~68peY0VTZ0654h8E84 z734hh&FHwC($_Y6yxk&<3h8w%O@BE2NlFgE_=#XXrXwh8vg3Sa^TE!u!san7IXs=b zB|B1+^6w+W zjvApm!Dnr+@TwL0oUfxeIpD%H-)7J;+Hssb zd2Wuxzns8Jl`9z`Fm7kavNN|{&2C#4o-4HXXxUkp!0tqHZs@5=b$&dAOM%(&J&?Cr zeumM)&7*Fmcavwxgm0HL#}d5w&E%`5MePDMJIvA7@Da^`sR({bVY?Ki`ENpo!;`NpS5{Z9DG ze)`h`DocAw4p)?Omm@MRgXk8XWX{@kJF$imPD)Nn0osZ3=~i=*OBj?#cKS-em~bXN z{%;2OYL8RWvgxL_G7~T-EjWUPy41Vy170zlfAYARK&|P5G1g*Ne!EjQ zsy0nJ-NH1JS6)4iJe|#Zv$xfU4RqE%t4u5>EUc%O*1!{ZO)yIN5by4u>0ggvpkD2t zcLDc4+}+LSzw@d7^BvW_L&XC1iJ%TO7w7NZ6erZ3h5AYVl|#k$pZqBh=fCr({&b;) zfzP3ClqgUPC;^lNNK+a)vCH$Bfp#C|H= zIWPIiAySGYE}>52q>`1Di8q}iiS(bKETzYI|_ITA31dG`( zOnKF$GB4xkoUJ}E(^1pt6BI`?pX(!#uj4nk?MLM*j)MA)*H+a_rDe5R#Dhi{*+~M( z<<|qg^&nkPz#U77vlAd>JY&ZklxQ=DS$)Ts9lUOi#Typbz+#K(y42NAJ)1kSkW#*f~*6DZ5unO~kCQIzCd!x9#V`#)OwVsVl*N%V!5=^<8uIL7Gf`Tozt z)^8$Y=9@KSvJq;$m}_UVtx4y>?;;-+5Uj984U8lET5jv^pt-seGd~OQo7`#3Fi_X} z7_0J{Z~7DO-eDwcMb2i}m%X?gK3}F7<9X#mzVCT_+ZguQ9+H8086GGrs#-+^E<`wj z?5x48m!JW^Wo@4h|DeZo!kq7;`FoKbPIm{pR*ZT@uY^BfeOa{?;rZF*#NB- zy83fEWmSqyCH)Tq>1^Lh@}P%6Bv-^g8gMBJ%{|l3@Fo5vi_S?^kU46h_O_Fz-xfPn z{μoTXP0o`4o>pX=w+)V0_dIQ(kahTWTtXl^*3akBO@L2Z(3Ji5 zgUF?urNfK3H?bbMB0x#3Ss`~)OZz>1bB5VLSH}AN^=NOV4!pu26-{4)TQb9oGCvR%lF(z(mdjhDxnm z53>SU?258>g7XdF*RJz3$hf8vn-X7{e%TjrnSlvWftd&{S9)dP^X3)i7J*=flg8_A z%}0tu<-A>fc>_IX<7^!@FEDpzf<-xBhae8FwF?&`AELE%JE>dBg|-CDgu1SRHxMM) zVjul@Z%{}l=r14B{$&jA)0Z|Jrp0rt56$n{PQK+G3h|Hz#bRJ?)&h7)nsgj6p3n)X zL8=SrG0j#6jH+tPuQmgyNy?+J%3p;}ybk5VtXiQtiisr1(OE$Az7#TNlo`hMqD>XM zD4kq3o)VlttuF5kSC@;Bm&QDj=AYJQ^}CKOGXE%q1|VZF#Oon@4Wz~P&Ln%NML)4^ zS!Lz0Igvgu|NSk*HoPQ(e4|4gkx+Gpe3l|g5vx+;8F+ZPs3MN*A!= zQ%Y1AEnVXJ+ z0jT)r7N!PN2WkMdps@l&2sB(^`+?|`;vR*aeFFUjr2ob4M^VVBu;9bOt)xnt&ahjO`r3Ko_7ZG=||0 z^ZqIZlXhigGQt>;rPsuA?X+kx)??WU<^4xmu}EHhzRlXS+nQV!R^iW zgMsbV(%Z%lR<639EkP|)RR^2i2b+4MF^ZU>c@}H$CI)^?G&l50jm5T@%Z0&t2K$+W z*-srtI|rb%Mv~)X7x{W&Y6ZhxeYA8sI*{u%pA+Kb0r2IsxY~M%r$va!u76vh6z9Hn z?q0OX$sbH<7fPFf&tpzWP6HduBVH6H&hivZlr#r76!C=N&xullnW|B$LS*IiV-jPo z-X`OU1zehW@sslbXawzr6%INB+T6**o^Ax;Q_1P^)0D9AG}BU3aM)O0R)_aAD4t|( zcAI=E%ENTx=6v}h96SGe$Pjat&Z2mu=S!0{4$~4Yu~{YM0#M9jj#%R7a@oy=nJQFZ zc`?O~0`orK(9A&PiWo|cRAON-S-Fgq0&ec8Fs6)0s zA(1gO%Whr+i8FGs{ljv2eeZ%x>6@zv%uRxm?*g8u;aeBk6?I>+*G~0x%Et7jJ)M&@ z5<3%3lC3%2jd>T?2Ozl?i597|>ZY56a)YUZVFRuSPqmTFOqp?w*LH z9}5a;xM_-Nmb|7kL&stlA_2fi;eaSbOhcT80g`ZX<@yRSGtK*K(-d_3!wuCY774dql=3G)VhVn*P>ApB}{5lkA zT!V?2A%pNamrRC->=yiJ)q$h9%N1S&m3%=AVm^7(g@pP+A=4yAr$kHPT@$y}B;;5L z*_-|u9j^WHS<(*tlh5b=AnD5e=BJ~DA%deAHT$eHz_dr@enCtaox+b@SV&3JT7tel zx*XNs%VGSAlcxW*3?QVxK0G5PfUm~4DpK9;`4)`?c)kfP{we78nUV|jMc z3zJW>iMa1APZiRC}&uU8`e$AO~MwOj)&)&z;iR{NVl&_W|MzbuxO&0S-MCkqTlCCQ%a#wg$C#=DA4Dma9Nsd&S51GjGlzW6^_02C+12Vof?OG^WBQE62;g|2f6gHV&%yfW##JC}}aQ$)rS_ef#rx|vE z_(Mc=(|e$vX5|7e;$WgsM|`}zSDRE%LFT~|hL0onzlFAwS6{Lj#6YxXXyszD`(V6&o-Z5t+rgy4$?DwFZm}H@|~_T#O?pE-rS6B zM$!(I+s$tSM0MP}ceYD1^RlYb-)>l45)k6buG?q%y9e8Z_|{)RA<0aB9$Nh7V**c3jHkKlP-R|$FkA0Qyh%JNuDph@)Nvk4W?P(fK!V2g6 z7aOm4zqe#81-wGEN$Cv!#7)P9@64ny$9m(N`?4b8v;2Yp@wEMM6{n(bT6ENhhQdb; zqaH{@Rm?kF0sQ4Cs!vKj^vf?x`Y%SETsEK^q1dE0sSM}p5VFd3^m@OeJupqt+{-xY z;>-dPJx}oFqRcJUG%L>Q*Z&9`HI|RP(a+7kU*#^=5HRZ$HumUAjAN;s`;Y}|P(*UK z$dh+M9`N|6UuDlub1u5ju0M9Mw5Bdu1aEbsgM+qHt$RMkBG6P0IGsjpT>@JejA!z< zrbN?igB~-ne%-Y+?3$b0a_Tf=nstAMB4j_eOw4#L#wabN&N!f1s>PRT2j=?B>s~ki#*`CK^1Kig+r-$Y+TX~<<2x6cyu_Nlahsud20W&V?rcB` zv+8c$bRGl#R0M(Sj@yYM{+U!aOMTYZDudKed< zKSeSoZbVGx8!J*UA^L(Ba^%X!^Ydy@k_s1Lm~3@X@qw~hEtO5mH8 z+GmLg)>EaN$0g=PhXzX|Q48?o-C~rd1aL2(J6H#DHY5-ozwymfJP3x{tPopL4>o(1Xp4VDMr^E)L67*J2ML>`u zGi~6Js%G5`-hu6{g zX)RHa;?5$ko}be5YU?@w7~pCe%H-omC!Y5YqCKLP>Jqb}oI$NvsG1{MTB=}MpnkJb zg1ALF`U>)XZQs7RS4mI-0x%3;TQ9SHO06ba`N{lb->1x zD^G9Svj^~7+9yYaV;N^rb=-w`uGlU5v^3 zIv!w?jXq@RyzM1M=a_rGkiLFRk5Qsiq_)Sv*e_=BvJPnvxru+m$8>4k4T1CqaJ5NA zBVkpOg(b`n8%(3!Rjcy!tx74H7+c-ITxKP#DKNRhZqm1=Y|6KH6-Tg}J^zhbwi+4Z z%R}ixH&e-g2rLK7u;4_#W=FfWn;(x)My!{d)I$B*(H$RBe7kJv+MgQr-_x?E+9rJe zd9l6k^OlOZUhVs+T&j!~i#(EQ3c1U-Y75L6NgVo&JaG%WJLsG=G2bABepgNDVIGOa z{!(n!X!5z4k_<6X=_w7g3_=bUD(ewOuHclRi_7RWYBMQMW|Udbu7cxW=M?(8OqXxj zmkS{~MU>7|lc|bzk1f4kR@O~PvVBS6C*RB0Q4S`=5ObqyKj3oMvRX}I zt|Buaz!XI0$8uBgvqPG}PQHO(B{u(xTY%st^-iPRI{_Ist*4q>#O+}m7ddX5B>aKl zBVKgI3-emIc~#>lqC@C2lw?;P2CSop$n%`Onj%|F=1mB-a`@eI2Il$oiQnXbsa5rA z8timuS2-f1IVDI2P76;kYpjk&r@Wi;zwqTYe_wkai)O1S^lTbax8D0=DzbYE3$w}r z{e0PI0ZqHp(~~x^NxlsZ{S-)Gx*AfxeUqJvM1nW#D!wUJ7ECdoXAC4nRDAbT5oL^= zn7`uTY1Wc%gMNwz8Bnm%d4~Pco55>q^%kU9=!&O<^`)4E_3bdyIYzB zc7BTEcJHH-*1|m-qtCXq{cA{&^SD=ievq#&JzDIod!q4J>Gsk~Av+)sAunpq8o2TkF2t*FoQR>ecvoMf`~LTXft)bci?OR&Nw zz3K|5`an7$4Ts}L1+vf}c3G6cjSoO;@BJcaay$7^M;MOvy^)jK+%U@v9Hg02+}EkkqW zlUf{0nWbrjdDvec(`VT?1ErYi8!(`=r(NXFv;HE zI6O#*u<4wdu}uMzRfvza#l6&Sw3{b$o=o(L$NK!z_Dg;JbJafVjrNVi;*M$O^zzIW zBAt1Q(d0c~kU`O5+ozo?ko=7DWOrW~;HIlaVLns2L*krTMPP^qWSy=S*W%g61v+DW z5>2?L*9s?5?%*DR^obg%JgBJlj2~_@m*0hJM!$DrW2{;;Ei0w#!+BN63^_i*$?z;a ztsCNa8_pN+V5&+YTlP^JzLj+uO{Q?{KG3Z_g~Aq=WdXm}?trc{A1sV;S$mKamvU*~ zlgJw%gMDsv28Y`h1Y4f@9-lW3V|L4ZCXhXG-EOnYH*e9hyU*HRLW!4qvsUgrm8A`IK__OZ4iygSpkFTRY8-eLUAg|DOF(&slt9 zo3ioT>U|}Laqs}46B4;lGHn))Ua0DxR zyihUrurlT`p%D^9%{9OKywem3oYMO18D&FT_9EhG#YXW0EnHPwUq@l)s~ru8N>u)V+5EwLerRB z?HsKDAZAV)C`v~YQ(h%#qVi9K&?^BNa|pzq7YKB9b!B#CW43cN1A=&Xcz`UdKJAr=~cg4sGT-_yhlO*{fxXaGUXEWkeo`s4V|!p;_OH?X0R zoijv$#@X4zgcodNXaweBV}p`u0{sG+vNEwiUrbzF?5td1LsL#}F4I4Tv;U3Ee;E(j z03DDG$_HqP7n+G};%p3d6rhn%kOfGX+c`mgrpW@J?BJmR{+j;(6;)8#{Ch4y1^SB( z{LntUa?mVPQ+H8l9=RYZ3y6z}1u7a26;@VWb`D-P7A6*MUKSR9;NN;eg%1T4f&K*w z^e<8V3(((s{swC2r~*|9K{H1?Q&WHoG-7%u2@sSS%&b30ya)T$iywF=qu;pmf6XsL zdpWuPnPWCKcPf3?k5|;r*x3e}A}1yK|3+c{uVrjv{3pTo&W_gi)?s1{1Y3jeXmf&! z83d)+#F*FA&e6sYs@sP4_Rv&5LkN`7K&VdA2=YT4Ar{aOyWqX;30p&c7OXAqY>C&} z(AG?V#*N7YY-;Fi4WSWKwY~3c@>3et7J@$u_i(_!aQz!u-B&?v8!9&7pDTwTD&YU< z^?!%xujLY&MIb=qdbb$nM+)2=!KTp6SXNHdy9Mj#3jobfVB-Lo0)EwT zf}oN3`zCfV0hUQZIYaPp7X7b>DZ9%%TNIu=fLwttMv#m)B5xv_Hn zgJw=PmVeZ7@UZ`b7B()He~iJw#raQK*f^j$?|-9@jr$*U+@OEZ2V!C8{yWVeR(7_3 z(89&_4<0}j?05M>91WqlvyRX|wgLc^Ej++b-2gxz9kPQy@B>YXQ33#^Y)$O|cNz_S z5J*ZCpv}+C0up0o%=H|^$;v7!#?8*fEiS^v!NMi@zxU9+d!EJ#V(19D TS4a>yCp#x9HMN+$IO_icwA6bT diff --git a/contrib/zlib/zlib.map b/contrib/zlib/zlib.map deleted file mode 100644 index 82ce98cf7..000000000 --- a/contrib/zlib/zlib.map +++ /dev/null @@ -1,94 +0,0 @@ -ZLIB_1.2.0 { - global: - compressBound; - deflateBound; - inflateBack; - inflateBackEnd; - inflateBackInit_; - inflateCopy; - local: - deflate_copyright; - inflate_copyright; - inflate_fast; - inflate_table; - zcalloc; - zcfree; - z_errmsg; - gz_error; - gz_intmax; - _*; -}; - -ZLIB_1.2.0.2 { - gzclearerr; - gzungetc; - zlibCompileFlags; -} ZLIB_1.2.0; - -ZLIB_1.2.0.8 { - deflatePrime; -} ZLIB_1.2.0.2; - -ZLIB_1.2.2 { - adler32_combine; - crc32_combine; - deflateSetHeader; - inflateGetHeader; -} ZLIB_1.2.0.8; - -ZLIB_1.2.2.3 { - deflateTune; - gzdirect; -} ZLIB_1.2.2; - -ZLIB_1.2.2.4 { - inflatePrime; -} ZLIB_1.2.2.3; - -ZLIB_1.2.3.3 { - adler32_combine64; - crc32_combine64; - gzopen64; - gzseek64; - gztell64; - inflateUndermine; -} ZLIB_1.2.2.4; - -ZLIB_1.2.3.4 { - inflateReset2; - inflateMark; -} ZLIB_1.2.3.3; - -ZLIB_1.2.3.5 { - gzbuffer; - gzoffset; - gzoffset64; - gzclose_r; - gzclose_w; -} ZLIB_1.2.3.4; - -ZLIB_1.2.5.1 { - deflatePending; -} ZLIB_1.2.3.5; - -ZLIB_1.2.5.2 { - deflateResetKeep; - gzgetc_; - inflateResetKeep; -} ZLIB_1.2.5.1; - -ZLIB_1.2.7.1 { - inflateGetDictionary; - gzvprintf; -} ZLIB_1.2.5.2; - -ZLIB_1.2.9 { - inflateCodesUsed; - inflateValidate; - uncompress2; - gzfread; - gzfwrite; - deflateGetDictionary; - adler32_z; - crc32_z; -} ZLIB_1.2.7.1; diff --git a/contrib/zlib/zlib.pc.in b/contrib/zlib/zlib.pc.in deleted file mode 100644 index 7e5acf9c7..000000000 --- a/contrib/zlib/zlib.pc.in +++ /dev/null @@ -1,13 +0,0 @@ -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -sharedlibdir=@sharedlibdir@ -includedir=@includedir@ - -Name: zlib -Description: zlib compression library -Version: @VERSION@ - -Requires: -Libs: -L${libdir} -L${sharedlibdir} -lz -Cflags: -I${includedir} diff --git a/contrib/zlib/zlib2ansi b/contrib/zlib/zlib2ansi deleted file mode 100644 index 15e3e165f..000000000 --- a/contrib/zlib/zlib2ansi +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/perl - -# Transform K&R C function definitions into ANSI equivalent. -# -# Author: Paul Marquess -# Version: 1.0 -# Date: 3 October 2006 - -# TODO -# -# Asumes no function pointer parameters. unless they are typedefed. -# Assumes no literal strings that look like function definitions -# Assumes functions start at the beginning of a line - -use strict; -use warnings; - -local $/; -$_ = <>; - -my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments - -my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; -my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; -my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; - - -while (s/^ - ( # Start $1 - ( # Start $2 - .*? # Minimal eat content - ( ^ \w [\w\s\*]+ ) # $3 -- function name - \s* # optional whitespace - ) # $2 - Matched up to before parameter list - - \( \s* # Literal "(" + optional whitespace - ( [^\)]+ ) # $4 - one or more anythings except ")" - \s* \) # optional whitespace surrounding a Literal ")" - - ( (?: $dList )+ ) # $5 - - $sp ^ { # literal "{" at start of line - ) # Remember to $1 - //xsom - ) -{ - my $all = $1 ; - my $prefix = $2; - my $param_list = $4 ; - my $params = $5; - - StripComments($params); - StripComments($param_list); - $param_list =~ s/^\s+//; - $param_list =~ s/\s+$//; - - my $i = 0 ; - my %pList = map { $_ => $i++ } - split /\s*,\s*/, $param_list; - my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; - - my @params = split /\s*;\s*/, $params; - my @outParams = (); - foreach my $p (@params) - { - if ($p =~ /,/) - { - my @bits = split /\s*,\s*/, $p; - my $first = shift @bits; - $first =~ s/^\s*//; - push @outParams, $first; - $first =~ /^(\w+\s*)/; - my $type = $1 ; - push @outParams, map { $type . $_ } @bits; - } - else - { - $p =~ s/^\s+//; - push @outParams, $p; - } - } - - - my %tmp = map { /$pMatch/; $_ => $pList{$1} } - @outParams ; - - @outParams = map { " $_" } - sort { $tmp{$a} <=> $tmp{$b} } - @outParams ; - - print $prefix ; - print "(\n" . join(",\n", @outParams) . ")\n"; - print "{" ; - -} - -# Output any trailing code. -print ; -exit 0; - - -sub StripComments -{ - - no warnings; - - # Strip C & C++ coments - # From the perlfaq - $_[0] =~ - - s{ - /\* ## Start of /* ... */ comment - [^*]*\*+ ## Non-* followed by 1-or-more *'s - ( - [^/*][^*]*\*+ - )* ## 0-or-more things which don't start with / - ## but do end with '*' - / ## End of /* ... */ comment - - | ## OR C++ Comment - // ## Start of C++ comment // - [^\n]* ## followed by 0-or-more non end of line characters - - | ## OR various things which aren't comments: - - ( - " ## Start of " ... " string - ( - \\. ## Escaped char - | ## OR - [^"\\] ## Non "\ - )* - " ## End of " ... " string - - | ## OR - - ' ## Start of ' ... ' string - ( - \\. ## Escaped char - | ## OR - [^'\\] ## Non '\ - )* - ' ## End of ' ... ' string - - | ## OR - - . ## Anything other char - [^/"'\\]* ## Chars which doesn't start a comment, string or escape - ) - }{$2}gxs; - -} From bf1aaf98f9d032a4c13d55d08e792c9d7cee3b97 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 10:51:48 +0300 Subject: [PATCH 225/305] IFCImporter: Add explicit instantiation of log_prefix so IFCMaterial.cpp can see it --- code/IFCLoader.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/code/IFCLoader.h b/code/IFCLoader.h index 4cf116f8e..000683bb8 100644 --- a/code/IFCLoader.h +++ b/code/IFCLoader.h @@ -60,6 +60,13 @@ namespace Assimp { class DB; } +#ifndef _MSC_VER + // GCC and Clang need to see this explicit declaration to avoid warning + // MSVC complains about redeclaration even though this is just the + // declaration, not the definition + class IFCImporter; + template<> const std::string LogFunctions::log_prefix; +#endif // _MSC_VER // ------------------------------------------------------------------------------------------- /** Load the IFC format, which is an open specification to describe building and construction From afce9842286a441d5a93a4920f6ce9ca9d3f488c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:05:27 +0300 Subject: [PATCH 226/305] FBXImporter: Add explicit instantiation of log_prefix so other FBX source files can see it --- code/FBXImporter.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/code/FBXImporter.h b/code/FBXImporter.h index 43be97ffa..54be8384e 100644 --- a/code/FBXImporter.h +++ b/code/FBXImporter.h @@ -58,6 +58,14 @@ namespace Formatter { typedef class basic_formatter< char, std::char_traits, std::allocator > format; } +#ifndef _MSC_VER +// GCC and Clang need to see this explicit declaration to avoid warning +// MSVC complains about redeclaration even though this is just the +// declaration, not the definition +class FBXImporter; +template<> const std::string LogFunctions::log_prefix; +#endif // _MSC_VER + // ------------------------------------------------------------------------------------------- /** Load the Autodesk FBX file format. From 65547d57606d9c9345b00d38271f3e499cee4001 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:10:22 +0300 Subject: [PATCH 227/305] Upgrade RapidJSON to get rid of a clang warning --- .../rapidjson/include/rapidjson/allocators.h | 42 +- .../rapidjson/include/rapidjson/document.h | 1073 +++++++-- .../include/rapidjson/encodedstream.h | 58 +- .../rapidjson/include/rapidjson/encodings.h | 233 +- .../rapidjson/include/rapidjson/error/en.h | 21 +- .../rapidjson/include/rapidjson/error/error.h | 27 +- .../include/rapidjson/filereadstream.h | 13 +- .../include/rapidjson/filewritestream.h | 17 +- contrib/rapidjson/include/rapidjson/fwd.h | 151 ++ .../include/rapidjson/internal/biginteger.h | 14 +- .../include/rapidjson/internal/diyfp.h | 19 +- .../include/rapidjson/internal/dtoa.h | 50 +- .../include/rapidjson/internal/ieee754.h | 3 +- .../include/rapidjson/internal/regex.h | 734 ++++++ .../include/rapidjson/internal/stack.h | 68 +- .../include/rapidjson/internal/strfunc.h | 32 +- .../include/rapidjson/internal/strtod.h | 39 +- .../include/rapidjson/internal/swap.h | 9 + .../include/rapidjson/istreamwrapper.h | 115 + .../include/rapidjson/memorybuffer.h | 2 +- .../include/rapidjson/memorystream.h | 16 +- .../include/rapidjson/msinttypes/stdint.h | 8 +- .../include/rapidjson/ostreamwrapper.h | 81 + contrib/rapidjson/include/rapidjson/pointer.h | 131 +- .../include/rapidjson/prettywriter.h | 98 +- .../rapidjson/include/rapidjson/rapidjson.h | 314 ++- contrib/rapidjson/include/rapidjson/reader.h | 1345 ++++++++--- contrib/rapidjson/include/rapidjson/schema.h | 2016 +++++++++++++++++ contrib/rapidjson/include/rapidjson/stream.h | 179 ++ .../include/rapidjson/stringbuffer.h | 30 +- contrib/rapidjson/include/rapidjson/writer.h | 468 +++- contrib/rapidjson/license.txt | 2 +- contrib/rapidjson/readme.md | 81 +- 33 files changed, 6467 insertions(+), 1022 deletions(-) create mode 100644 contrib/rapidjson/include/rapidjson/fwd.h create mode 100644 contrib/rapidjson/include/rapidjson/internal/regex.h create mode 100644 contrib/rapidjson/include/rapidjson/istreamwrapper.h create mode 100644 contrib/rapidjson/include/rapidjson/ostreamwrapper.h create mode 100644 contrib/rapidjson/include/rapidjson/schema.h create mode 100644 contrib/rapidjson/include/rapidjson/stream.h diff --git a/contrib/rapidjson/include/rapidjson/allocators.h b/contrib/rapidjson/include/rapidjson/allocators.h index d74a67155..655f4a385 100644 --- a/contrib/rapidjson/include/rapidjson/allocators.h +++ b/contrib/rapidjson/include/rapidjson/allocators.h @@ -179,9 +179,10 @@ public: size = RAPIDJSON_ALIGN(size); if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) - AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); + if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) + return NULL; - void *buffer = reinterpret_cast(chunkHead_ + 1) + chunkHead_->size; + void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; chunkHead_->size += size; return buffer; } @@ -194,14 +195,16 @@ public: if (newSize == 0) return NULL; + originalSize = RAPIDJSON_ALIGN(originalSize); + newSize = RAPIDJSON_ALIGN(newSize); + // Do not shrink if new size is smaller than original if (originalSize >= newSize) return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { + if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { size_t increment = static_cast(newSize - originalSize); - increment = RAPIDJSON_ALIGN(increment); if (chunkHead_->size + increment <= chunkHead_->capacity) { chunkHead_->size += increment; return originalPtr; @@ -209,11 +212,13 @@ public: } // Realloc process: allocate and copy memory, do not free original buffer. - void* newBuffer = Malloc(newSize); - RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. - if (originalSize) - std::memcpy(newBuffer, originalPtr, originalSize); - return newBuffer; + if (void* newBuffer = Malloc(newSize)) { + if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; + } + else + return NULL; } //! Frees a memory block (concept Allocator) @@ -227,15 +232,20 @@ private: //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. + \return true if success. */ - void AddChunk(size_t capacity) { + bool AddChunk(size_t capacity) { if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); - ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity)); - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); + if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + return true; + } + else + return false; } static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. diff --git a/contrib/rapidjson/include/rapidjson/document.h b/contrib/rapidjson/include/rapidjson/document.h index c6acbd907..93b091f64 100644 --- a/contrib/rapidjson/include/rapidjson/document.h +++ b/contrib/rapidjson/include/rapidjson/document.h @@ -20,40 +20,29 @@ #include "reader.h" #include "internal/meta.h" #include "internal/strfunc.h" +#include "memorystream.h" +#include "encodedstream.h" #include // placement new +#include +RAPIDJSON_DIAG_PUSH #ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -#elif defined(__GNUC__) -RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +#ifdef __GNUC__ RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 6 +RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions #endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_HAS_STDSTRING - -#ifndef RAPIDJSON_HAS_STDSTRING -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation -#else -#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default -#endif -/*! \def RAPIDJSON_HAS_STDSTRING - \ingroup RAPIDJSON_CONFIG - \brief Enable RapidJSON support for \c std::string - - By defining this preprocessor symbol to \c 1, several convenience functions for using - \ref rapidjson::GenericValue with \c std::string are enabled, especially - for construction and comparison. - - \hideinitializer -*/ -#endif // !defined(RAPIDJSON_HAS_STDSTRING) - -#if RAPIDJSON_HAS_STDSTRING -#include -#endif // RAPIDJSON_HAS_STDSTRING +#endif // __GNUC__ #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::iterator, std::random_access_iterator_tag @@ -69,6 +58,9 @@ RAPIDJSON_NAMESPACE_BEGIN template class GenericValue; +template +class GenericDocument; + //! Name-value pair in a JSON object value. /*! This class was internal to GenericValue. It used to be a inner struct. @@ -155,6 +147,7 @@ public: Otherwise, the copy constructor is implicitly defined. */ GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } //! @name stepping //@{ @@ -257,6 +250,7 @@ struct GenericStringRef { typedef CharType Ch; //!< character type of the string //! Create string reference from \c const character array +#ifndef __clang__ // -Wdocumentation /*! This constructor implicitly creates a constant string reference from a \c const character array. It has better performance than @@ -279,11 +273,13 @@ struct GenericStringRef { In such cases, the referenced string should be \b copied to the GenericValue instead. */ +#endif template GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT : s(str), length(N-1) {} //! Explicitly create string reference from \c const character pointer +#ifndef __clang__ // -Wdocumentation /*! This constructor can be used to \b explicitly create a reference to a constant string pointer. @@ -302,18 +298,23 @@ struct GenericStringRef { In such cases, the referenced string should be \b copied to the GenericValue instead. */ +#endif explicit GenericStringRef(const CharType* str) - : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != NULL); } + : s(str), length(NotNullStrLen(str)) {} //! Create constant string reference from pointer and length +#ifndef __clang__ // -Wdocumentation /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \param len length of the string, excluding the trailing NULL terminator \post \ref s == str && \ref length == len \note Constant complexity. */ +#endif GenericStringRef(const CharType* str, SizeType len) - : s(str), length(len) { RAPIDJSON_ASSERT(s != NULL); } + : s(RAPIDJSON_LIKELY(str) ? str : emptyString), length(len) { RAPIDJSON_ASSERT(str != 0 || len == 0u); } + + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} //! implicit conversion to plain CharType pointer operator const Ch *() const { return s; } @@ -322,13 +323,24 @@ struct GenericStringRef { const SizeType length; //!< length of the string (excluding the trailing NULL terminator) private: - //! Disallow copy-assignment - GenericStringRef operator=(const GenericStringRef&); + SizeType NotNullStrLen(const CharType* str) { + RAPIDJSON_ASSERT(str != 0); + return internal::StrLen(str); + } + + /// Empty string - used when passing in a NULL pointer + static const Ch emptyString[]; + //! Disallow construction from non-const array template GenericStringRef(CharType (&str)[N]) /* = delete */; + //! Copy assignment operator not permitted - immutable type + GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */; }; +template +const CharType GenericStringRef::emptyString[] = { CharType() }; + //! Mark a character pointer as constant string /*! Mark a plain character pointer as a "string literal". This function can be used to avoid copying a character string to be referenced as a @@ -343,7 +355,7 @@ private: */ template inline GenericStringRef StringRef(const CharType* str) { - return GenericStringRef(str, internal::StrLen(str)); + return GenericStringRef(str); } //! Mark a character pointer as constant string @@ -401,6 +413,127 @@ template struct IsGenericValue : IsGenericValueImpl::Type {}; } // namespace internal +/////////////////////////////////////////////////////////////////////////////// +// TypeHelper + +namespace internal { + +template +struct TypeHelper {}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsBool(); } + static bool Get(const ValueType& v) { return v.GetBool(); } + static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } + static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static int Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt64(); } + static int64_t Get(const ValueType& v) { return v.GetInt64(); } + static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } + static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint64(); } + static uint64_t Get(const ValueType& v) { return v.GetUint64(); } + static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } + static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsDouble(); } + static double Get(const ValueType& v) { return v.GetDouble(); } + static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } + static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsFloat(); } + static float Get(const ValueType& v) { return v.GetFloat(); } + static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } + static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } +}; + +template +struct TypeHelper { + typedef const typename ValueType::Ch* StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } + static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; + +#if RAPIDJSON_HAS_STDSTRING +template +struct TypeHelper > { + typedef std::basic_string StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } + static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; +#endif + +template +struct TypeHelper { + typedef typename ValueType::Array ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(ValueType& v) { return v.GetArray(); } + static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } + static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstArray ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(const ValueType& v) { return v.GetArray(); } +}; + +template +struct TypeHelper { + typedef typename ValueType::Object ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(ValueType& v) { return v.GetObject(); } + static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstObject ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(const ValueType& v) { return v.GetObject(); } +}; + +} // namespace internal + +// Forward declarations +template class GenericArray; +template class GenericObject; + /////////////////////////////////////////////////////////////////////////////// // GenericValue @@ -428,17 +561,21 @@ public: typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. typedef GenericValue ValueType; //!< Value type of itself. + typedef GenericArray Array; + typedef GenericArray ConstArray; + typedef GenericObject Object; + typedef GenericObject ConstObject; //!@name Constructors and destructor. //@{ //! Default constructor creates a null value. - GenericValue() RAPIDJSON_NOEXCEPT : data_(), flags_(kNullFlag) {} + GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 - GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_), flags_(rhs.flags_) { - rhs.flags_ = kNullFlag; // give up contents + GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { + rhs.data_.f.flags = kNullFlag; // give up contents } #endif @@ -446,6 +583,16 @@ private: //! Copy constructor is not permitted. GenericValue(const GenericValue& rhs); +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Moving from a GenericDocument is not permitted. + template + GenericValue(GenericDocument&& rhs); + + //! Move assignment from a GenericDocument is not permitted. + template + GenericValue& operator=(GenericDocument&& rhs); +#endif + public: //! Constructor with JSON value type. @@ -453,13 +600,13 @@ public: \param type Type of the value. \note Default content for number is zero. */ - explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_(), flags_() { - static const unsigned defaultFlags[7] = { + explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { + static const uint16_t defaultFlags[7] = { kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, kNumberAnyFlag }; - RAPIDJSON_ASSERT(type <= kNumberType); - flags_ = defaultFlags[type]; + RAPIDJSON_ASSERT(type >= kNullType && type <= kNumberType); + data_.f.flags = defaultFlags[type]; // Use ShortString to store empty string. if (type == kStringType) @@ -471,10 +618,50 @@ public: \tparam SourceAllocator allocator of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) \see CopyFrom() */ - template< typename SourceAllocator > - GenericValue(const GenericValue& rhs, Allocator & allocator); + template + GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + switch (rhs.GetType()) { + case kObjectType: { + SizeType count = rhs.data_.o.size; + Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); + } + data_.f.flags = kObjectFlag; + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } + break; + case kArrayType: { + SizeType count = rhs.data_.a.size; + GenericValue* le = reinterpret_cast(allocator.Malloc(count * sizeof(GenericValue))); + const GenericValue* re = rhs.GetElementsPointer(); + for (SizeType i = 0; i < count; i++) + new (&le[i]) GenericValue(re[i], allocator, copyConstStrings); + data_.f.flags = kArrayFlag; + data_.a.size = data_.a.capacity = count; + SetElementsPointer(le); + } + break; + case kStringType: + if (rhs.data_.f.flags == kConstStringFlag && !copyConstStrings) { + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + } + else + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } + } //! Constructor for boolean value. /*! \param b Boolean value @@ -484,96 +671,125 @@ public: */ #ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen template - explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 #else explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT #endif - : data_(), flags_(b ? kTrueFlag : kFalseFlag) { + : data_() { // safe-guard against failing SFINAE RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + data_.f.flags = b ? kTrueFlag : kFalseFlag; } //! Constructor for int value. - explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberIntFlag) { + explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { data_.n.i64 = i; - if (i >= 0) - flags_ |= kUintFlag | kUint64Flag; + data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; } //! Constructor for unsigned value. - explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUintFlag) { + explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { data_.n.u64 = u; - if (!(u & 0x80000000)) - flags_ |= kIntFlag | kInt64Flag; + data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); } //! Constructor for int64_t value. - explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberInt64Flag) { + explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { data_.n.i64 = i64; + data_.f.flags = kNumberInt64Flag; if (i64 >= 0) { - flags_ |= kNumberUint64Flag; + data_.f.flags |= kNumberUint64Flag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - flags_ |= kUintFlag; + data_.f.flags |= kUintFlag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; + data_.f.flags |= kIntFlag; } else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; + data_.f.flags |= kIntFlag; } //! Constructor for uint64_t value. - explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUint64Flag) { + explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { data_.n.u64 = u64; + data_.f.flags = kNumberUint64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) - flags_ |= kInt64Flag; + data_.f.flags |= kInt64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - flags_ |= kUintFlag; + data_.f.flags |= kUintFlag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; + data_.f.flags |= kIntFlag; } //! Constructor for double value. - explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberDoubleFlag) { data_.n.d = d; } + explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for float value. + explicit GenericValue(float f) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = static_cast(f); data_.f.flags = kNumberDoubleFlag; } //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(StringRef(s, length)); } + GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } //! Constructor for constant string (i.e. do not make a copy of string) - explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(s); } + explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s, length), allocator); } + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } + GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Constructor for copy-string from a string object (i.e. do make a copy of string) /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue(const std::basic_string& s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } + GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #endif + //! Constructor for Array. + /*! + \param a An array obtained by \c GetArray(). + \note \c Array is always pass-by-value. + \note the source array is moved into this value and the sourec array becomes empty. + */ + GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { + a.value_.data_ = Data(); + a.value_.data_.f.flags = kArrayFlag; + } + + //! Constructor for Object. + /*! + \param o An object obtained by \c GetObject(). + \note \c Object is always pass-by-value. + \note the source object is moved into this value and the sourec object becomes empty. + */ + GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { + o.value_.data_ = Data(); + o.value_.data_.f.flags = kObjectFlag; + } + //! Destructor. /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(flags_) { + switch(data_.f.flags) { case kArrayFlag: - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(data_.a.elements); + { + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(e); + } break; case kObjectFlag: for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) m->~Member(); - Allocator::Free(data_.o.members); + Allocator::Free(GetMembersPointer()); break; case kCopyStringFlag: - Allocator::Free(const_cast(data_.s.str)); + Allocator::Free(const_cast(GetStringPointer())); break; default: @@ -638,12 +854,13 @@ public: \tparam SourceAllocator Allocator type of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator to use for copying + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) */ template - GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { - RAPIDJSON_ASSERT((void*)this != (void const*)&rhs); + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); this->~GenericValue(); - new (this) GenericValue(rhs, allocator); + new (this) GenericValue(rhs, allocator, copyConstStrings); return *this; } @@ -660,6 +877,20 @@ public: return *this; } + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.value, b.value); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + //! Prepare Value for move semantics /*! \return *this */ GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } @@ -709,7 +940,7 @@ public: else return data_.n.u64 == rhs.data_.n.u64; - default: // kTrueType, kFalseType, kNullType + default: return true; } } @@ -757,20 +988,58 @@ public: //!@name Type //@{ - Type GetType() const { return static_cast(flags_ & kTypeMask); } - bool IsNull() const { return flags_ == kNullFlag; } - bool IsFalse() const { return flags_ == kFalseFlag; } - bool IsTrue() const { return flags_ == kTrueFlag; } - bool IsBool() const { return (flags_ & kBoolFlag) != 0; } - bool IsObject() const { return flags_ == kObjectFlag; } - bool IsArray() const { return flags_ == kArrayFlag; } - bool IsNumber() const { return (flags_ & kNumberFlag) != 0; } - bool IsInt() const { return (flags_ & kIntFlag) != 0; } - bool IsUint() const { return (flags_ & kUintFlag) != 0; } - bool IsInt64() const { return (flags_ & kInt64Flag) != 0; } - bool IsUint64() const { return (flags_ & kUint64Flag) != 0; } - bool IsDouble() const { return (flags_ & kDoubleFlag) != 0; } - bool IsString() const { return (flags_ & kStringFlag) != 0; } + Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } + bool IsNull() const { return data_.f.flags == kNullFlag; } + bool IsFalse() const { return data_.f.flags == kFalseFlag; } + bool IsTrue() const { return data_.f.flags == kTrueFlag; } + bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } + bool IsObject() const { return data_.f.flags == kObjectFlag; } + bool IsArray() const { return data_.f.flags == kArrayFlag; } + bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } + bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } + bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } + bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } + bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } + bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } + bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } + + // Checks whether a number can be losslessly converted to a double. + bool IsLosslessDouble() const { + if (!IsNumber()) return false; + if (IsUint64()) { + uint64_t u = GetUint64(); + volatile double d = static_cast(u); + return (d >= 0.0) + && (d < static_cast((std::numeric_limits::max)())) + && (u == static_cast(d)); + } + if (IsInt64()) { + int64_t i = GetInt64(); + volatile double d = static_cast(i); + return (d >= static_cast((std::numeric_limits::min)())) + && (d < static_cast((std::numeric_limits::max)())) + && (i == static_cast(d)); + } + return true; // double, int, uint are always lossless + } + + // Checks whether a number is a float (possible lossy). + bool IsFloat() const { + if ((data_.f.flags & kDoubleFlag) == 0) + return false; + double d = GetDouble(); + return d >= -3.4028234e38 && d <= 3.4028234e38; + } + // Checks whether a number can be losslessly converted to a float. + bool IsLosslessFloat() const { + if (!IsNumber()) return false; + double a = GetDouble(); + if (a < static_cast(-(std::numeric_limits::max)()) + || a > static_cast((std::numeric_limits::max)())) + return false; + double b = static_cast(static_cast(a)); + return a >= b && a <= b; // Prevent -Wfloat-equal + } //@} @@ -784,7 +1053,7 @@ public: //!@name Bool //@{ - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return flags_ == kTrueFlag; } + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } //!< Set boolean value /*! \post IsBool() == true */ GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } @@ -837,8 +1106,14 @@ public: return member->value; else { RAPIDJSON_ASSERT(false); // see above note - static GenericValue NullValue; - return NullValue; + + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); } } template @@ -852,16 +1127,16 @@ public: //! Const member iterator /*! \pre IsObject() == true */ - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members); } + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } //! Const \em past-the-end member iterator /*! \pre IsObject() == true */ - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members + data_.o.size); } + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } //! Member iterator /*! \pre IsObject() == true */ - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members); } + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } //! \em Past-the-end member iterator /*! \pre IsObject() == true */ - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members + data_.o.size); } + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } //! Check whether a member exists in the object. /*! @@ -949,8 +1224,8 @@ public: \return Iterator to member, if it exists. Otherwise returns \ref MemberEnd(). */ - MemberIterator FindMember(const std::basic_string& name) { return FindMember(StringRef(name)); } - ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(StringRef(name)); } + MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } #endif //! Add a member (name-value pair) to the object. @@ -967,20 +1242,21 @@ public: RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - Object& o = data_.o; + ObjectData& o = data_.o; if (o.size >= o.capacity) { if (o.capacity == 0) { o.capacity = kDefaultObjectCapacity; - o.members = reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member))); + SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); } else { SizeType oldCapacity = o.capacity; o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - o.members = reinterpret_cast(allocator.Realloc(o.members, oldCapacity * sizeof(Member), o.capacity * sizeof(Member))); + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); } } - o.members[o.size].name.RawAssign(name); - o.members[o.size].value.RawAssign(value); + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); o.size++; return *this; } @@ -1159,18 +1435,14 @@ public: MemberIterator RemoveMember(MemberIterator m) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - MemberIterator last(data_.o.members + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) { - // Move the last one to this place - *m = *last; - } - else { - // Only one left, just destroy - m->~Member(); - } + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy --data_.o.size; return m; } @@ -1200,7 +1472,7 @@ public: MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); @@ -1208,11 +1480,39 @@ public: MemberIterator pos = MemberBegin() + (first - MemberBegin()); for (MemberIterator itr = pos; itr != last; ++itr) itr->~Member(); - std::memmove(&*pos, &*last, (MemberEnd() - last) * sizeof(Member)); - data_.o.size -= (last - first); + std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); return pos; } + //! Erase a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note Linear time complexity. + */ + bool EraseMember(const Ch* name) { + GenericValue n(StringRef(name)); + return EraseMember(n); + } + +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } +#endif + + template + bool EraseMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + EraseMember(m); + return true; + } + else + return false; + } + + Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + //@} //!@name Array @@ -1220,7 +1520,7 @@ public: //! Set this value as an empty array. /*! \post IsArray == true */ - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } //! Get the number of elements in array. SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } @@ -1237,8 +1537,9 @@ public: */ void Clear() { RAPIDJSON_ASSERT(IsArray()); - for (SizeType i = 0; i < data_.a.size; ++i) - data_.a.elements[i].~GenericValue(); + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); data_.a.size = 0; } @@ -1250,16 +1551,16 @@ public: GenericValue& operator[](SizeType index) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(index < data_.a.size); - return data_.a.elements[index]; + return GetElementsPointer()[index]; } const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } //! Element iterator /*! \pre IsArray() == true */ - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements; } + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } //! \em Past-the-end element iterator /*! \pre IsArray() == true */ - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements + data_.a.size; } + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } //! Constant element iterator /*! \pre IsArray() == true */ ConstValueIterator Begin() const { return const_cast(*this).Begin(); } @@ -1276,7 +1577,7 @@ public: GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsArray()); if (newCapacity > data_.a.capacity) { - data_.a.elements = (GenericValue*)allocator.Realloc(data_.a.elements, data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)); + SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); data_.a.capacity = newCapacity; } return *this; @@ -1296,7 +1597,7 @@ public: RAPIDJSON_ASSERT(IsArray()); if (data_.a.size >= data_.a.capacity) Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); - data_.a.elements[data_.a.size++].RawAssign(value); + GetElementsPointer()[data_.a.size++].RawAssign(value); return *this; } @@ -1350,7 +1651,7 @@ public: GenericValue& PopBack() { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(!Empty()); - data_.a.elements[--data_.a.size].~GenericValue(); + GetElementsPointer()[--data_.a.size].~GenericValue(); return *this; } @@ -1376,35 +1677,48 @@ public: ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(data_.a.size > 0); - RAPIDJSON_ASSERT(data_.a.elements != 0); + RAPIDJSON_ASSERT(GetElementsPointer() != 0); RAPIDJSON_ASSERT(first >= Begin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= End()); ValueIterator pos = Begin() + (first - Begin()); for (ValueIterator itr = pos; itr != last; ++itr) itr->~GenericValue(); - std::memmove(pos, last, (End() - last) * sizeof(GenericValue)); - data_.a.size -= (last - first); + std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); return pos; } + Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } + ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } + //@} //!@name Number //@{ - int GetInt() const { RAPIDJSON_ASSERT(flags_ & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(flags_ & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(flags_ & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(flags_ & kUint64Flag); return data_.n.u64; } + int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + //! Get the value as double type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. + */ double GetDouble() const { RAPIDJSON_ASSERT(IsNumber()); - if ((flags_ & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((flags_ & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((flags_ & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((flags_ & kInt64Flag) != 0) return (double)data_.n.i64; // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((flags_ & kUint64Flag) != 0); return (double)data_.n.u64; // uint64_t -> double (may lose precision) + if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) + } + + //! Get the value as float type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. + */ + float GetFloat() const { + return static_cast(GetDouble()); } GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } @@ -1412,18 +1726,19 @@ public: GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(static_cast(f)); return *this; } //@} //!@name String //@{ - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? data_.ss.str : data_.s.str); } + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. @@ -1450,7 +1765,7 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { return SetString(StringRef(s, length), allocator); } //! Set this value as a string by copying from source string. /*! \param s source string. @@ -1458,7 +1773,15 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(StringRef(s), allocator); } + + //! Set this value as a string by copying from source string. + /*! \param s source string reference + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(StringRefType s, Allocator& allocator) { this->~GenericValue(); SetStringRaw(s, allocator); return *this; } #if RAPIDJSON_HAS_STDSTRING //! Set this value as a string by copying from source string. @@ -1468,11 +1791,35 @@ public: \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(StringRef(s), allocator); } #endif //@} + //!@name Array + //@{ + + //! Templated version for checking whether this value is type T. + /*! + \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string + */ + template + bool Is() const { return internal::TypeHelper::Is(*this); } + + template + T Get() const { return internal::TypeHelper::Get(*this); } + + template + T Get() { return internal::TypeHelper::Get(*this); } + + template + ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } + + template + ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } + + //@} + //! Generate events of this value to a Handler. /*! This function adopts the GoF visitor pattern. Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. @@ -1488,35 +1835,35 @@ public: case kTrueType: return handler.Bool(true); case kObjectType: - if (!handler.StartObject()) + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) return false; for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. - if (!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.flags_ & kCopyFlag) != 0)) + if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) return false; - if (!m->value.Accept(handler)) + if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) return false; } return handler.EndObject(data_.o.size); case kArrayType: - if (!handler.StartArray()) + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) return false; - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - if (!v->Accept(handler)) + for (const GenericValue* v = Begin(); v != End(); ++v) + if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) return false; return handler.EndArray(data_.a.size); case kStringType: - return handler.String(GetString(), GetStringLength(), (flags_ & kCopyFlag) != 0); + return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); default: RAPIDJSON_ASSERT(GetType() == kNumberType); - if (IsInt()) return handler.Int(data_.n.i.i); + if (IsDouble()) return handler.Double(data_.n.d); + else if (IsInt()) return handler.Int(data_.n.i.i); else if (IsUint()) return handler.Uint(data_.n.u.u); else if (IsInt64()) return handler.Int64(data_.n.i64); - else if (IsUint64()) return handler.Uint64(data_.n.u64); - else return handler.Double(data_.n.d); + else return handler.Uint64(data_.n.u64); } } @@ -1525,16 +1872,16 @@ private: template friend class GenericDocument; enum { - kBoolFlag = 0x100, - kNumberFlag = 0x200, - kIntFlag = 0x400, - kUintFlag = 0x800, - kInt64Flag = 0x1000, - kUint64Flag = 0x2000, - kDoubleFlag = 0x4000, - kStringFlag = 0x100000, - kCopyFlag = 0x200000, - kInlineStrFlag = 0x400000, + kBoolFlag = 0x0008, + kNumberFlag = 0x0010, + kIntFlag = 0x0020, + kUintFlag = 0x0040, + kInt64Flag = 0x0080, + kUint64Flag = 0x0100, + kDoubleFlag = 0x0200, + kStringFlag = 0x0400, + kCopyFlag = 0x0800, + kInlineStrFlag = 0x1000, // Initial flags of different types. kNullFlag = kNullType, @@ -1552,16 +1899,27 @@ private: kObjectFlag = kObjectType, kArrayFlag = kArrayType, - kTypeMask = 0xFF // bitwise-and with mask of 0xFF can be optimized by compiler + kTypeMask = 0x07 }; static const SizeType kDefaultArrayCapacity = 16; static const SizeType kDefaultObjectCapacity = 16; + struct Flag { +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION + char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer +#elif RAPIDJSON_64BIT + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes +#else + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes +#endif + uint16_t flags; + }; + struct String { - const Ch* str; SizeType length; - unsigned hashcode; //!< reserved + SizeType hashcode; //!< reserved + const Ch* str; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars @@ -1570,15 +1928,15 @@ private: // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as // the string terminator as well. For getting the string length back from that value just use // "MaxSize - str[LenPos]". - // This allows to store 11-chars strings in 32-bit mode and 15-chars strings in 64-bit mode - // inline (for `UTF8`-encoded strings). + // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, + // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). struct ShortString { - enum { MaxChars = sizeof(String) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; Ch str[MaxChars]; - inline static bool Usable(SizeType len) { return (MaxSize >= len); } - inline void SetLength(SizeType len) { str[LenPos] = (Ch)(MaxSize - len); } - inline SizeType GetLength() const { return (SizeType)(MaxSize - str[LenPos]); } + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } + inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // By using proper binary layout, retrieval of different integer types do not need conversions. @@ -1607,69 +1965,79 @@ private: double d; }; // 8 bytes - struct Object { - Member* members; + struct ObjectData { SizeType size; SizeType capacity; + Member* members; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - struct Array { - GenericValue* elements; + struct ArrayData { SizeType size; SizeType capacity; + GenericValue* elements; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode union Data { String s; ShortString ss; Number n; - Object o; - Array a; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + ObjectData o; + ArrayData a; + Flag f; + }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } + RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } + RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } + RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } + RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } + RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { - flags_ = kArrayFlag; + data_.f.flags = kArrayFlag; if (count) { - data_.a.elements = (GenericValue*)allocator.Malloc(count * sizeof(GenericValue)); - std::memcpy(data_.a.elements, values, count * sizeof(GenericValue)); + GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); + std::memcpy(e, values, count * sizeof(GenericValue)); } else - data_.a.elements = NULL; + SetElementsPointer(0); data_.a.size = data_.a.capacity = count; } //! Initialize this value as object with initial data, without calling destructor. void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { - flags_ = kObjectFlag; + data_.f.flags = kObjectFlag; if (count) { - data_.o.members = (Member*)allocator.Malloc(count * sizeof(Member)); - std::memcpy(data_.o.members, members, count * sizeof(Member)); + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); + std::memcpy(m, members, count * sizeof(Member)); } else - data_.o.members = NULL; + SetMembersPointer(0); data_.o.size = data_.o.capacity = count; } //! Initialize this value as constant string, without calling destructor. void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { - flags_ = kConstStringFlag; - data_.s.str = s; + data_.f.flags = kConstStringFlag; + SetStringPointer(s); data_.s.length = s.length; } //! Initialize this value as copy string with initial data, without calling destructor. void SetStringRaw(StringRefType s, Allocator& allocator) { - Ch* str = NULL; - if(ShortString::Usable(s.length)) { - flags_ = kShortStringFlag; + Ch* str = 0; + if (ShortString::Usable(s.length)) { + data_.f.flags = kShortStringFlag; data_.ss.SetLength(s.length); str = data_.ss.str; } else { - flags_ = kCopyStringFlag; + data_.f.flags = kCopyStringFlag; data_.s.length = s.length; - str = (Ch *)allocator.Malloc((s.length + 1) * sizeof(Ch)); - data_.s.str = str; + str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); + SetStringPointer(str); } std::memcpy(str, s, s.length * sizeof(Ch)); str[s.length] = '\0'; @@ -1678,8 +2046,8 @@ private: //! Assignment without calling destructor void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { data_ = rhs.data_; - flags_ = rhs.flags_; - rhs.flags_ = kNullFlag; + // data_.f.flags = rhs.data_.f.flags; + rhs.data_.f.flags = kNullFlag; } template @@ -1699,7 +2067,6 @@ private: } Data data_; - unsigned flags_; }; //! GenericValue with UTF8 encoding @@ -1724,7 +2091,22 @@ public: typedef Allocator AllocatorType; //!< Allocator type from template parameter. //! Constructor - /*! \param allocator Optional allocator for allocating memory. + /*! Creates an empty document of specified type. + \param type Mandatory type of object to create. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + } + + //! Constructor + /*! Creates an empty document which type is Null. + \param allocator Optional allocator for allocating memory. \param stackCapacity Optional initial capacity of stack in bytes. \param stackAllocator Optional allocator for allocating memory for stack. */ @@ -1732,13 +2114,13 @@ public: allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - : ValueType(std::move(rhs)), + : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), stack_(std::move(rhs.stack_)), @@ -1778,6 +2160,54 @@ public: } #endif + //! Exchange the contents of this document with those of another. + /*! + \param rhs Another document. + \note Constant complexity. + \see GenericValue::Swap + */ + GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { + ValueType::Swap(rhs); + stack_.Swap(rhs.stack_); + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(parseResult_, rhs.parseResult_); + return *this; + } + + // Allow Swap with ValueType. + // Refer to Effective C++ 3rd Edition/Item 33: Avoid hiding inherited names. + using ValueType::Swap; + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.doc, b.doc); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Populate this document by a generator which produces SAX events. + /*! \tparam Generator A functor with bool f(Handler) prototype. + \param g Generator functor which sends SAX events to the parameter. + \return The document itself for fluent API. + */ + template + GenericDocument& Populate(Generator& g) { + ClearStackOnExit scope(*this); + if (g(*this)) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + //!@name Parse from stream //!@{ @@ -1790,13 +2220,13 @@ public: */ template GenericDocument& ParseStream(InputStream& is) { - ValueType::SetNull(); // Remove existing root if exist - GenericReader reader(&stack_.GetAllocator()); + GenericReader reader( + stack_.HasAllocator() ? &stack_.GetAllocator() : 0); ClearStackOnExit scope(*this); parseResult_ = reader.template Parse(is, *this); if (parseResult_) { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - this->RawAssign(*stack_.template Pop(1)); // Add this-> to prevent issue 13. + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document } return *this; } @@ -1855,7 +2285,7 @@ public: \param str Read-only zero-terminated string to be parsed. */ template - GenericDocument& Parse(const Ch* str) { + GenericDocument& Parse(const typename SourceEncoding::Ch* str) { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); GenericStringStream s(str); return ParseStream(s); @@ -1876,6 +2306,42 @@ public: GenericDocument& Parse(const Ch* str) { return Parse(str); } + + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + MemoryStream ms(reinterpret_cast(str), length * sizeof(typename SourceEncoding::Ch)); + EncodedInputStream is(ms); + ParseStream(is); + return *this; + } + + template + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + template + GenericDocument& Parse(const std::basic_string& str) { + // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) + return Parse(str.c_str()); + } + + template + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str.c_str()); + } + + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str); + } +#endif // RAPIDJSON_HAS_STDSTRING + //!@} //!@name Handling parse errors @@ -1890,10 +2356,26 @@ public: //! Get the position of last parsing error in input, 0 otherwise. size_t GetErrorOffset() const { return parseResult_.Offset(); } + //! Implicit conversion to get the last parse result +#ifndef __clang // -Wdocumentation + /*! \return \ref ParseResult of the last parse operation + + \code + Document doc; + ParseResult ok = doc.Parse(json); + if (!ok) + printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); + \endcode + */ +#endif + operator ParseResult() const { return parseResult_; } //!@} //! Get the allocator of this document. - Allocator& GetAllocator() { return *allocator_; } + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } //! Get the capacity of stack in bytes. size_t GetStackCapacity() const { return stack_.GetCapacity(); } @@ -1910,9 +2392,10 @@ private: }; // callers of the following private Handler functions - template friend class GenericReader; // for parsing + // template friend class GenericReader; // for parsing template friend class GenericValue; // for deep copying +public: // Implementation of Handler bool Null() { new (stack_.template Push()) ValueType(); return true; } bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } @@ -1922,6 +2405,14 @@ private: bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + bool RawNumber(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + bool String(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); @@ -1936,7 +2427,7 @@ private: bool EndObject(SizeType memberCount) { typename ValueType::Member* members = stack_.template Pop(memberCount); - stack_.template Top()->SetObjectRaw(members, (SizeType)memberCount, GetAllocator()); + stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); return true; } @@ -1977,38 +2468,146 @@ private: //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; -// defined here due to the dependency on GenericDocument -template -template -inline -GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) -{ - switch (rhs.GetType()) { - case kObjectType: - case kArrayType: { // perform deep copy via SAX Handler - GenericDocument d(&allocator); - rhs.Accept(d); - RawAssign(*d.stack_.template Pop(1)); - } - break; - case kStringType: - if (rhs.flags_ == kConstStringFlag) { - flags_ = rhs.flags_; - data_ = *reinterpret_cast(&rhs.data_); - } else { - SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); - } - break; - default: // kNumberType, kTrueType, kFalseType, kNullType - flags_ = rhs.flags_; - data_ = *reinterpret_cast(&rhs.data_); - } -} +//! Helper class for accessing Value of array type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetArray(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericArray { +public: + typedef GenericArray ConstArray; + typedef GenericArray Array; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef ValueType* ValueIterator; // This may be const or non-const iterator + typedef const ValueT* ConstValueIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; -RAPIDJSON_NAMESPACE_END + template + friend class GenericValue; -#if defined(_MSC_VER) || defined(__GNUC__) -RAPIDJSON_DIAG_POP + GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} + GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } + ~GenericArray() {} + + SizeType Size() const { return value_.Size(); } + SizeType Capacity() const { return value_.Capacity(); } + bool Empty() const { return value_.Empty(); } + void Clear() const { value_.Clear(); } + ValueType& operator[](SizeType index) const { return value_[index]; } + ValueIterator Begin() const { return value_.Begin(); } + ValueIterator End() const { return value_.End(); } + GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } + GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PopBack() const { value_.PopBack(); return *this; } + ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + ValueIterator begin() const { return value_.Begin(); } + ValueIterator end() const { return value_.End(); } #endif +private: + GenericArray(); + GenericArray(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +//! Helper class for accessing Value of object type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetObject(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericObject { +public: + typedef GenericObject ConstObject; + typedef GenericObject Object; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator + typedef GenericMemberIterator ConstMemberIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename ValueType::Ch Ch; + + template + friend class GenericValue; + + GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} + GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } + ~GenericObject() {} + + SizeType MemberCount() const { return value_.MemberCount(); } + bool ObjectEmpty() const { return value_.ObjectEmpty(); } + template ValueType& operator[](T* name) const { return value_[name]; } + template ValueType& operator[](const GenericValue& name) const { return value_[name]; } +#if RAPIDJSON_HAS_STDSTRING + ValueType& operator[](const std::basic_string& name) const { return value_[name]; } +#endif + MemberIterator MemberBegin() const { return value_.MemberBegin(); } + MemberIterator MemberEnd() const { return value_.MemberEnd(); } + bool HasMember(const Ch* name) const { return value_.HasMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } +#endif + template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } + MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } + template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } +#if RAPIDJSON_HAS_STDSTRING + MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } +#endif + GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_STDSTRING + GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + void RemoveAllMembers() { value_.RemoveAllMembers(); } + bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } +#endif + template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } + MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } + MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } + bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } +#endif + template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + MemberIterator begin() const { return value_.MemberBegin(); } + MemberIterator end() const { return value_.MemberEnd(); } +#endif + +private: + GenericObject(); + GenericObject(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + #endif // RAPIDJSON_DOCUMENT_H_ diff --git a/contrib/rapidjson/include/rapidjson/encodedstream.h b/contrib/rapidjson/include/rapidjson/encodedstream.h index 7c8863fee..223601c05 100644 --- a/contrib/rapidjson/include/rapidjson/encodedstream.h +++ b/contrib/rapidjson/include/rapidjson/encodedstream.h @@ -15,13 +15,19 @@ #ifndef RAPIDJSON_ENCODEDSTREAM_H_ #define RAPIDJSON_ENCODEDSTREAM_H_ -#include "rapidjson.h" +#include "stream.h" +#include "memorystream.h" #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Input byte stream wrapper with a statically bound encoding. @@ -57,10 +63,38 @@ private: Ch current_; }; +//! Specialized for UTF8 MemoryStream. +template <> +class EncodedInputStream, MemoryStream> { +public: + typedef UTF8<>::Ch Ch; + + EncodedInputStream(MemoryStream& is) : is_(is) { + if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); + } + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) {} + void Flush() {} + Ch* PutBegin() { return 0; } + size_t PutEnd(Ch*) { return 0; } + + MemoryStream& is_; + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); +}; + //! Output byte stream wrapper with statically bound encoding. /*! \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam InputByteStream Type of input byte stream. For example, FileWriteStream. + \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. */ template class EncodedOutputStream { @@ -77,8 +111,8 @@ public: void Flush() { os_.Flush(); } // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); } - Ch Take() { RAPIDJSON_ASSERT(false); } + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } @@ -142,11 +176,11 @@ private: // FF FE UTF-16LE // EF BB BF UTF-8 - const unsigned char* c = (const unsigned char *)is_->Peek4(); + const unsigned char* c = reinterpret_cast(is_->Peek4()); if (!c) return; - unsigned bom = c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24); + unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); hasBOM_ = false; if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } @@ -166,7 +200,7 @@ private: // xx xx xx xx UTF-8 if (!hasBOM_) { - unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); switch (pattern) { case 0x08: type_ = kUTF32BE; break; case 0x0A: type_ = kUTF16BE; break; @@ -193,7 +227,7 @@ private: //! Output stream wrapper with dynamically bound encoding and automatic encoding detection. /*! \tparam CharType Type of character for writing. - \tparam InputByteStream type of output byte stream to be wrapped. + \tparam OutputByteStream type of output byte stream to be wrapped. */ template class AutoUTFOutputStream { @@ -227,8 +261,8 @@ public: void Flush() { os_->Flush(); } // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); } - Ch Take() { RAPIDJSON_ASSERT(false); } + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } @@ -254,6 +288,10 @@ private: RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/encodings.h b/contrib/rapidjson/include/rapidjson/encodings.h index 90b46ed32..0df1c3435 100644 --- a/contrib/rapidjson/include/rapidjson/encodings.h +++ b/contrib/rapidjson/include/rapidjson/encodings.h @@ -120,19 +120,45 @@ struct UTF8 { } } + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + PutUnsafe(os, static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + } + template static bool Decode(InputStream& is, unsigned* codepoint) { -#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | ((unsigned char)c & 0x3Fu) -#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) +#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) - Ch c = is.Take(); + typename InputStream::Ch c = is.Take(); if (!(c & 0x80)) { - *codepoint = (unsigned char)c; + *codepoint = static_cast(c); return true; } - unsigned char type = GetRange((unsigned char)c); - *codepoint = (0xFF >> type) & (unsigned char)c; + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) { + *codepoint = 0; + } else { + *codepoint = (0xFFu >> type) & static_cast(c); + } bool result = true; switch (type) { case 2: TAIL(); return result; @@ -152,7 +178,7 @@ struct UTF8 { template static bool Validate(InputStream& is, OutputStream& os) { #define COPY() os.Put(c = is.Take()) -#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) Ch c; COPY(); @@ -160,7 +186,7 @@ struct UTF8 { return true; bool result = true; - switch (GetRange((unsigned char)c)) { + switch (GetRange(static_cast(c))) { case 2: TAIL(); return result; case 3: TAIL(); TAIL(); return result; case 4: COPY(); TRANS(0x50); TAIL(); return result; @@ -196,12 +222,12 @@ struct UTF8 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - Ch c = Take(is); - if ((unsigned char)c != 0xEFu) return c; + typename InputByteStream::Ch c = Take(is); + if (static_cast(c) != 0xEFu) return c; c = is.Take(); - if ((unsigned char)c != 0xBBu) return c; + if (static_cast(c) != 0xBBu) return c; c = is.Take(); - if ((unsigned char)c != 0xBFu) return c; + if (static_cast(c) != 0xBFu) return c; c = is.Take(); return c; } @@ -209,13 +235,15 @@ struct UTF8 { template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return is.Take(); + return static_cast(is.Take()); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xEFu); os.Put(0xBBu); os.Put(0xBFu); + os.Put(static_cast(0xEFu)); + os.Put(static_cast(0xBBu)); + os.Put(static_cast(0xBFu)); } template @@ -255,22 +283,38 @@ struct UTF16 { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; os.Put(static_cast((v >> 10) | 0xD800)); - os.Put((v & 0x3FF) | 0xDC00); + os.Put(static_cast((v & 0x3FF) | 0xDC00)); + } + } + + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + PutUnsafe(os, static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + PutUnsafe(os, static_cast((v >> 10) | 0xD800)); + PutUnsafe(os, static_cast((v & 0x3FF) | 0xDC00)); } } template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - Ch c = is.Take(); + typename InputStream::Ch c = is.Take(); if (c < 0xD800 || c > 0xDFFF) { - *codepoint = c; + *codepoint = static_cast(c); return true; } else if (c <= 0xDBFF) { - *codepoint = (c & 0x3FF) << 10; + *codepoint = (static_cast(c) & 0x3FF) << 10; c = is.Take(); - *codepoint |= (c & 0x3FF); + *codepoint |= (static_cast(c) & 0x3FF); *codepoint += 0x10000; return c >= 0xDC00 && c <= 0xDFFF; } @@ -281,8 +325,8 @@ struct UTF16 { static bool Validate(InputStream& is, OutputStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - Ch c; - os.Put(c = is.Take()); + typename InputStream::Ch c; + os.Put(static_cast(c = is.Take())); if (c < 0xD800 || c > 0xDFFF) return true; else if (c <= 0xDBFF) { @@ -300,28 +344,29 @@ struct UTF16LE : UTF16 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned short)c == 0xFEFFu ? Take(is) : c; + return static_cast(c) == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take(); - c |= (unsigned char)is.Take() << 8; - return c; + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFFu); os.Put(0xFEu); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(c & 0xFFu); - os.Put((c >> 8) & 0xFFu); + os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); } }; @@ -332,28 +377,29 @@ struct UTF16BE : UTF16 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned short)c == 0xFEFFu ? Take(is) : c; + return static_cast(c) == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take(); - return c; + unsigned c = static_cast(static_cast(is.Take())) << 8; + c |= static_cast(is.Take()); + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFEu); os.Put(0xFFu); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put((c >> 8) & 0xFFu); - os.Put(c & 0xFFu); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(static_cast(static_cast(c) & 0xFFu)); } }; @@ -382,6 +428,13 @@ struct UTF32 { os.Put(codepoint); } + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, codepoint); + } + template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); @@ -406,32 +459,35 @@ struct UTF32LE : UTF32 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned)c == 0x0000FEFFu ? Take(is) : c; + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take(); - c |= (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take() << 16; - c |= (unsigned char)is.Take() << 24; - return c; + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 24; + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFFu); os.Put(0xFEu); os.Put(0x00u); os.Put(0x00u); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(c & 0xFFu); - os.Put((c >> 8) & 0xFFu); - os.Put((c >> 16) & 0xFFu); - os.Put((c >> 24) & 0xFFu); + os.Put(static_cast(c & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 24) & 0xFFu)); } }; @@ -442,32 +498,35 @@ struct UTF32BE : UTF32 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned)c == 0x0000FEFFu ? Take(is) : c; + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take() << 24; - c |= (unsigned char)is.Take() << 16; - c |= (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take(); - return c; + unsigned c = static_cast(static_cast(is.Take())) << 24; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0x00u); os.Put(0x00u); os.Put(0xFEu); os.Put(0xFFu); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put((c >> 24) & 0xFFu); - os.Put((c >> 16) & 0xFFu); - os.Put((c >> 8) & 0xFFu); - os.Put(c & 0xFFu); + os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast(c & 0xFFu)); } }; @@ -491,31 +550,37 @@ struct ASCII { os.Put(static_cast(codepoint & 0xFF)); } + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + PutUnsafe(os, static_cast(codepoint & 0xFF)); + } + template static bool Decode(InputStream& is, unsigned* codepoint) { - unsigned char c = static_cast(is.Take()); + uint8_t c = static_cast(is.Take()); *codepoint = c; return c <= 0X7F; } template static bool Validate(InputStream& is, OutputStream& os) { - unsigned char c = is.Take(); - os.Put(c); + uint8_t c = static_cast(is.Take()); + os.Put(static_cast(c)); return c <= 0x7F; } template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - Ch c = Take(is); - return c; + uint8_t c = static_cast(Take(is)); + return static_cast(c); } template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return is.Take(); + return static_cast(is.Take()); } template @@ -555,21 +620,28 @@ struct AutoUTF { #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x template - RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { + static RAPIDJSON_FORCEINLINE void Encode(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; (*f[os.GetType()])(os, codepoint); } + template + static RAPIDJSON_FORCEINLINE void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; + (*f[os.GetType()])(os, codepoint); + } + template - RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { + static RAPIDJSON_FORCEINLINE bool Decode(InputStream& is, unsigned* codepoint) { typedef bool (*DecodeFunc)(InputStream&, unsigned*); static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; return (*f[is.GetType()])(is, codepoint); } template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { typedef bool (*ValidateFunc)(InputStream&, OutputStream&); static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; return (*f[is.GetType()])(is, os); @@ -586,7 +658,7 @@ template struct Transcoder { //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; @@ -594,31 +666,50 @@ struct Transcoder { return true; } + template + static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::EncodeUnsafe(os, codepoint); + return true; + } + //! Validate one Unicode codepoint from an encoded stream. template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { return Transcode(is, os); // Since source/target encoding is different, must transcode. } }; +// Forward declaration. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c); + //! Specialization of Transcoder with same source and target encoding. template struct Transcoder { template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { return Encoding::Validate(is, os); // source/target encoding are the same } }; RAPIDJSON_NAMESPACE_END -#if defined(__GNUC__) || defined(_MSV_VER) +#if defined(__GNUC__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/error/en.h b/contrib/rapidjson/include/rapidjson/error/en.h index d5f9caab8..2db838bff 100644 --- a/contrib/rapidjson/include/rapidjson/error/en.h +++ b/contrib/rapidjson/include/rapidjson/error/en.h @@ -12,11 +12,17 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#ifndef RAPIDJSON_ERROR_EN_H__ -#define RAPIDJSON_ERROR_EN_H__ +#ifndef RAPIDJSON_ERROR_EN_H_ +#define RAPIDJSON_ERROR_EN_H_ #include "error.h" +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(covered-switch-default) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Maps error code of parsing into error message. @@ -32,7 +38,7 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); - case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); + case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); @@ -55,11 +61,14 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); - default: - return RAPIDJSON_ERROR_STRING("Unknown error."); + default: return RAPIDJSON_ERROR_STRING("Unknown error."); } } RAPIDJSON_NAMESPACE_END -#endif // RAPIDJSON_ERROR_EN_H__ +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_EN_H_ diff --git a/contrib/rapidjson/include/rapidjson/error/error.h b/contrib/rapidjson/include/rapidjson/error/error.h index f9094fb95..9311d2f03 100644 --- a/contrib/rapidjson/include/rapidjson/error/error.h +++ b/contrib/rapidjson/include/rapidjson/error/error.h @@ -12,11 +12,16 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#ifndef RAPIDJSON_ERROR_ERROR_H__ -#define RAPIDJSON_ERROR_ERROR_H__ +#ifndef RAPIDJSON_ERROR_ERROR_H_ +#define RAPIDJSON_ERROR_ERROR_H_ #include "../rapidjson.h" +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + /*! \file error.h */ /*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ @@ -99,7 +104,9 @@ enum ParseErrorCode { \see GenericReader::Parse, GenericDocument::Parse */ struct ParseResult { - + //!! Unspecified boolean type + typedef bool (ParseResult::*BooleanType)() const; +public: //! Default constructor, no error. ParseResult() : code_(kParseErrorNone), offset_(0) {} //! Constructor to set an error. @@ -110,8 +117,8 @@ struct ParseResult { //! Get the error offset, if \ref IsError(), 0 otherwise. size_t Offset() const { return offset_; } - //! Conversion to \c bool, returns \c true, iff !\ref IsError(). - operator bool() const { return !IsError(); } + //! Explicit conversion to \c bool, returns \c true, iff !\ref IsError(). + operator BooleanType() const { return !IsError() ? &ParseResult::IsError : NULL; } //! Whether the result is an error. bool IsError() const { return code_ != kParseErrorNone; } @@ -119,6 +126,10 @@ struct ParseResult { bool operator==(ParseErrorCode code) const { return code_ == code; } friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + bool operator!=(const ParseResult& that) const { return !(*this == that); } + bool operator!=(ParseErrorCode code) const { return !(*this == code); } + friend bool operator!=(ParseErrorCode code, const ParseResult & err) { return err != code; } + //! Reset error code. void Clear() { Set(kParseErrorNone); } //! Update error code and offset. @@ -143,4 +154,8 @@ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); RAPIDJSON_NAMESPACE_END -#endif // RAPIDJSON_ERROR_ERROR_H__ +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_ERROR_H_ diff --git a/contrib/rapidjson/include/rapidjson/filereadstream.h b/contrib/rapidjson/include/rapidjson/filereadstream.h index 3913eb74b..b56ea13b3 100644 --- a/contrib/rapidjson/include/rapidjson/filereadstream.h +++ b/contrib/rapidjson/include/rapidjson/filereadstream.h @@ -15,9 +15,16 @@ #ifndef RAPIDJSON_FILEREADSTREAM_H_ #define RAPIDJSON_FILEREADSTREAM_H_ -#include "rapidjson.h" +#include "stream.h" #include +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! File byte stream for input using fread(). @@ -85,4 +92,8 @@ private: RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_FILESTREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/filewritestream.h b/contrib/rapidjson/include/rapidjson/filewritestream.h index dfb9cbd02..6378dd60e 100644 --- a/contrib/rapidjson/include/rapidjson/filewritestream.h +++ b/contrib/rapidjson/include/rapidjson/filewritestream.h @@ -15,9 +15,14 @@ #ifndef RAPIDJSON_FILEWRITESTREAM_H_ #define RAPIDJSON_FILEWRITESTREAM_H_ -#include "rapidjson.h" +#include "stream.h" #include +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Wrapper of C file stream for input using fread(). @@ -57,7 +62,11 @@ public: void Flush() { if (current_ != buffer_) { - fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } current_ = buffer_; } } @@ -88,4 +97,8 @@ inline void PutN(FileWriteStream& stream, char c, size_t n) { RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_FILESTREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/fwd.h b/contrib/rapidjson/include/rapidjson/fwd.h new file mode 100644 index 000000000..e8104e841 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/fwd.h @@ -0,0 +1,151 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FWD_H_ +#define RAPIDJSON_FWD_H_ + +#include "rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN + +// encodings.h + +template struct UTF8; +template struct UTF16; +template struct UTF16BE; +template struct UTF16LE; +template struct UTF32; +template struct UTF32BE; +template struct UTF32LE; +template struct ASCII; +template struct AutoUTF; + +template +struct Transcoder; + +// allocators.h + +class CrtAllocator; + +template +class MemoryPoolAllocator; + +// stream.h + +template +struct GenericStringStream; + +typedef GenericStringStream > StringStream; + +template +struct GenericInsituStringStream; + +typedef GenericInsituStringStream > InsituStringStream; + +// stringbuffer.h + +template +class GenericStringBuffer; + +typedef GenericStringBuffer, CrtAllocator> StringBuffer; + +// filereadstream.h + +class FileReadStream; + +// filewritestream.h + +class FileWriteStream; + +// memorybuffer.h + +template +struct GenericMemoryBuffer; + +typedef GenericMemoryBuffer MemoryBuffer; + +// memorystream.h + +struct MemoryStream; + +// reader.h + +template +struct BaseReaderHandler; + +template +class GenericReader; + +typedef GenericReader, UTF8, CrtAllocator> Reader; + +// writer.h + +template +class Writer; + +// prettywriter.h + +template +class PrettyWriter; + +// document.h + +template +struct GenericMember; + +template +class GenericMemberIterator; + +template +struct GenericStringRef; + +template +class GenericValue; + +typedef GenericValue, MemoryPoolAllocator > Value; + +template +class GenericDocument; + +typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; + +// pointer.h + +template +class GenericPointer; + +typedef GenericPointer Pointer; + +// schema.h + +template +class IGenericRemoteSchemaDocumentProvider; + +template +class GenericSchemaDocument; + +typedef GenericSchemaDocument SchemaDocument; +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +template < + typename SchemaDocumentType, + typename OutputHandler, + typename StateAllocator> +class GenericSchemaValidator; + +typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/biginteger.h b/contrib/rapidjson/include/rapidjson/internal/biginteger.h index 99a30acf6..9d3e88c99 100644 --- a/contrib/rapidjson/include/rapidjson/internal/biginteger.h +++ b/contrib/rapidjson/include/rapidjson/internal/biginteger.h @@ -19,6 +19,7 @@ #if defined(_MSC_VER) && defined(_M_AMD64) #include // for _umul128 +#pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN @@ -50,7 +51,16 @@ public: if (length > 0) AppendDecimal64(decimals + i, decimals + i + length); } - + + BigInteger& operator=(const BigInteger &rhs) + { + if (this != &rhs) { + count_ = rhs.count_; + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + return *this; + } + BigInteger& operator=(uint64_t u) { digits_[0] = u; count_ = 1; @@ -230,7 +240,7 @@ private: uint64_t r = 0; for (const char* p = begin; p != end; ++p) { RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10 + (*p - '0'); + r = r * 10u + static_cast(*p - '0'); } return r; } diff --git a/contrib/rapidjson/include/rapidjson/internal/diyfp.h b/contrib/rapidjson/include/rapidjson/internal/diyfp.h index 3b6c4238c..29abf8046 100644 --- a/contrib/rapidjson/include/rapidjson/internal/diyfp.h +++ b/contrib/rapidjson/include/rapidjson/internal/diyfp.h @@ -21,9 +21,10 @@ #include "../rapidjson.h" -#if defined(_MSC_VER) && defined(_M_AMD64) +#if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) #include #pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN @@ -34,8 +35,13 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + struct DiyFp { - DiyFp() {} + DiyFp() : f(), e() {} DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} @@ -232,8 +238,8 @@ inline DiyFp GetCachedPower(int e, int* K) { } inline DiyFp GetCachedPower10(int exp, int *outExp) { - unsigned index = (exp + 348) / 8; - *outExp = -348 + index * 8; + unsigned index = (static_cast(exp) + 348u) / 8u; + *outExp = -348 + static_cast(index) * 8; return GetCachedPowerByIndex(index); } @@ -241,6 +247,11 @@ inline DiyFp GetCachedPower10(int exp, int *outExp) { RAPIDJSON_DIAG_POP #endif +#ifdef __clang__ +RAPIDJSON_DIAG_POP +RAPIDJSON_DIAG_OFF(padded) +#endif + } // namespace internal RAPIDJSON_NAMESPACE_END diff --git a/contrib/rapidjson/include/rapidjson/internal/dtoa.h b/contrib/rapidjson/include/rapidjson/internal/dtoa.h index 2d8d2e46a..bf2e9b2e5 100644 --- a/contrib/rapidjson/include/rapidjson/internal/dtoa.h +++ b/contrib/rapidjson/include/rapidjson/internal/dtoa.h @@ -29,6 +29,7 @@ namespace internal { #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) +RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 #endif inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { @@ -40,7 +41,7 @@ inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uin } } -inline unsigned CountDecimalDigit32(uint32_t n) { +inline int CountDecimalDigit32(uint32_t n) { // Simple pure C++ implementation was faster than __builtin_clz version in this situation. if (n < 10) return 1; if (n < 100) return 2; @@ -101,7 +102,8 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff kappa--; if (p2 < delta) { *K += kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-kappa]); + int index = -kappa; + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); return; } } @@ -145,10 +147,10 @@ inline char* WriteExponent(int K, char* buffer) { return buffer; } -inline char* Prettify(char* buffer, int length, int k) { +inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { const int kk = length + k; // 10^(kk-1) <= v < 10^kk - if (length <= kk && kk <= 21) { + if (0 <= k && kk <= 21) { // 1234e7 -> 12340000000 for (int i = length; i < kk; i++) buffer[i] = '0'; @@ -158,19 +160,44 @@ inline char* Prettify(char* buffer, int length, int k) { } else if (0 < kk && kk <= 21) { // 1234e-2 -> 12.34 - std::memmove(&buffer[kk + 1], &buffer[kk], length - kk); + std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); buffer[kk] = '.'; - return &buffer[length + 1]; + if (0 > k + maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[kk + 2]; // Reserve one zero + } + else + return &buffer[length + 1]; } else if (-6 < kk && kk <= 0) { // 1234e-6 -> 0.001234 const int offset = 2 - kk; - std::memmove(&buffer[offset], &buffer[0], length); + std::memmove(&buffer[offset], &buffer[0], static_cast(length)); buffer[0] = '0'; buffer[1] = '.'; for (int i = 2; i < offset; i++) buffer[i] = '0'; - return &buffer[length + offset]; + if (length - kk > maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = maxDecimalPlaces + 1; i > 2; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[3]; // Reserve one zero + } + else + return &buffer[length + offset]; + } + else if (kk < -maxDecimalPlaces) { + // Truncate to zero + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; } else if (length == 1) { // 1e30 @@ -179,14 +206,15 @@ inline char* Prettify(char* buffer, int length, int k) { } else { // 1234e30 -> 1.234e33 - std::memmove(&buffer[2], &buffer[1], length - 1); + std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); buffer[1] = '.'; buffer[length + 1] = 'e'; return WriteExponent(kk - 1, &buffer[0 + length + 2]); } } -inline char* dtoa(double value, char* buffer) { +inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { + RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); Double d(value); if (d.IsZero()) { if (d.Sign()) @@ -203,7 +231,7 @@ inline char* dtoa(double value, char* buffer) { } int length, K; Grisu2(value, buffer, &length, &K); - return Prettify(buffer, length, K); + return Prettify(buffer, length, K, maxDecimalPlaces); } } diff --git a/contrib/rapidjson/include/rapidjson/internal/ieee754.h b/contrib/rapidjson/include/rapidjson/internal/ieee754.h index e3f03364c..c2684ba2a 100644 --- a/contrib/rapidjson/include/rapidjson/internal/ieee754.h +++ b/contrib/rapidjson/include/rapidjson/internal/ieee754.h @@ -40,6 +40,7 @@ public: bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } + bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } @@ -47,7 +48,7 @@ public: int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } - static unsigned EffectiveSignificandSize(int order) { + static int EffectiveSignificandSize(int order) { if (order >= -1021) return 53; else if (order <= -1074) diff --git a/contrib/rapidjson/include/rapidjson/internal/regex.h b/contrib/rapidjson/include/rapidjson/internal/regex.h new file mode 100644 index 000000000..e1a2faae5 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/internal/regex.h @@ -0,0 +1,734 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_REGEX_H_ +#define RAPIDJSON_INTERNAL_REGEX_H_ + +#include "../allocators.h" +#include "../stream.h" +#include "stack.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 7 +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +#ifndef RAPIDJSON_REGEX_VERBOSE +#define RAPIDJSON_REGEX_VERBOSE 0 +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// DecodedStream + +template +class DecodedStream { +public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + +private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericRegex + +static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 +static const SizeType kRegexInvalidRange = ~SizeType(0); + +template +class GenericRegexSearch; + +//! Regular expression engine with subset of ECMAscript grammar. +/*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html +*/ +template +class GenericRegex { +public: + typedef Encoding EncodingType; + typedef typename Encoding::Ch Ch; + template friend class GenericRegexSearch; + + GenericRegex(const Ch* source, Allocator* allocator = 0) : + states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + anchorBegin_(), anchorEnd_() + { + GenericStringStream ss(source); + DecodedStream, Encoding> ds(ss); + Parse(ds); + } + + ~GenericRegex() {} + + bool IsValid() const { + return root_ != kRegexInvalidState; + } + +private: + enum Operator { + kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis + }; + + static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' + static const unsigned kRangeCharacterClass = 0xFFFFFFFE; + static const unsigned kRangeNegationFlag = 0x80000000; + + struct Range { + unsigned start; // + unsigned end; + SizeType next; + }; + + struct State { + SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; + }; + + struct Frag { + Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; + }; + + State& GetState(SizeType index) { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + const State& GetState(SizeType index) const { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + Range& GetRange(SizeType index) { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + const Range& GetRange(SizeType index) const { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + template + void Parse(DecodedStream& ds) { + Allocator allocator; + Stack operandStack(&allocator, 256); // Frag + Stack operatorStack(&allocator, 256); // Operator + Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) { + switch (codepoint = ds.Take()) { + case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { + unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') { + ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; + } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { + SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); + } + } + + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) { + Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; + +#if RAPIDJSON_REGEX_VERBOSE + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) { + State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); + } + printf("\n"); +#endif + } + } + + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { + State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; + } + + void PushOperand(Stack& operandStack, unsigned codepoint) { + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); + } + + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { + if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; + } + + SizeType Append(SizeType l1, SizeType l2) { + SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; + } + + void Patch(SizeType l, SizeType s) { + for (SizeType next; l != kRegexInvalidState; l = next) { + next = GetState(l).out; + GetState(l).out = s; + } + } + + bool Eval(Stack& operandStack, Operator op) { + switch (op) { + case kConcatenation: + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); + return true; + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + default: + RAPIDJSON_ASSERT(op == kOneOrMore); + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + } + } + + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { + RAPIDJSON_ASSERT(n <= m); + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) { + if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else { + Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? + } + return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) { + CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; + } + + static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } + + void CloneTopOperand(Stack& operandStack) { + const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) { + if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; + } + + template + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') { + if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; + } + + template + bool ParseRange(DecodedStream& ds, SizeType* range) { + bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) { + if (isBegin) { + isBegin = false; + if (codepoint == '^') { + negate = true; + continue; + } + } + + switch (codepoint) { + case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) { // Add trailing '-' + SizeType r = NewRange('-'); + RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; + } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; + + case '\\': + if (ds.Peek() == 'b') { + ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) { + case 1: + if (codepoint == '-') { + step++; + break; + } + // fall through to step 0 for other characters + + case 0: + { + SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; + } + } + } + return false; + } + + SizeType NewRange(unsigned codepoint) { + Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; + } + + template + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + unsigned codepoint; + switch (codepoint = ds.Take()) { + case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } + } + + Stack states_; + Stack ranges_; + SizeType root_; + SizeType stateCount_; + SizeType rangeCount_; + + static const unsigned kInfinityQuantifier = ~0u; + + // For SearchWithAnchoring() + bool anchorBegin_; + bool anchorEnd_; +}; + +template +class GenericRegexSearch { +public: + typedef typename RegexType::EncodingType Encoding; + typedef typename Encoding::Ch Ch; + + GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : + regex_(regex), allocator_(allocator), ownAllocator_(0), + state0_(allocator, 0), state1_(allocator, 0), stateSet_() + { + RAPIDJSON_ASSERT(regex_.IsValid()); + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); + state0_.template Reserve(regex_.stateCount_); + state1_.template Reserve(regex_.stateCount_); + } + + ~GenericRegexSearch() { + Allocator::Free(stateSet_); + RAPIDJSON_DELETE(ownAllocator_); + } + + template + bool Match(InputStream& is) { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) { + return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); + } + + bool Search(const Ch* s) { + GenericStringStream is(s); + return Search(is); + } + +private: + typedef typename RegexType::State State; + typedef typename RegexType::Range Range; + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, regex_.root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = regex_.GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == RegexType::kAnyCharacterClass || + (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, regex_.root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (regex_.stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) { + RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = regex_.GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { + stateSet_[index >> 5] |= (1u << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = regex_.GetRange(rangeIndex); + if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + const RegexType& regex_; + Allocator* allocator_; + Allocator* ownAllocator_; + Stack state0_; + Stack state1_; + uint32_t* stateSet_; +}; + +typedef GenericRegex > Regex; +typedef GenericRegexSearch RegexSearch; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/stack.h b/contrib/rapidjson/include/rapidjson/internal/stack.h index 722d56923..5c5398c35 100644 --- a/contrib/rapidjson/include/rapidjson/internal/stack.h +++ b/contrib/rapidjson/include/rapidjson/internal/stack.h @@ -15,7 +15,13 @@ #ifndef RAPIDJSON_INTERNAL_STACK_H_ #define RAPIDJSON_INTERNAL_STACK_H_ -#include "../rapidjson.h" +#include "../allocators.h" +#include "swap.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -32,7 +38,6 @@ public: // Optimization note: Do not allocate memory for stack_ in constructor. // Do it lazily when first Push() -> Expand() -> Resize(). Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { - RAPIDJSON_ASSERT(stackCapacity > 0); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -81,6 +86,15 @@ public: } #endif + void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(stack_, rhs.stack_); + internal::Swap(stackTop_, rhs.stackTop_); + internal::Swap(stackEnd_, rhs.stackEnd_); + internal::Swap(initialCapacity_, rhs.initialCapacity_); + } + void Clear() { stackTop_ = stack_; } void ShrinkToFit() { @@ -98,11 +112,22 @@ public: // Optimization note: try to minimize the size of this function for force inline. // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. template - RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { // Expand the stack if needed - if (stackTop_ + sizeof(T) * count >= stackEnd_) + if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) Expand(count); + } + template + RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + Reserve(count); + return PushUnsafe(count); + } + + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { + RAPIDJSON_ASSERT(stackTop_); + RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; @@ -122,9 +147,32 @@ public: } template - T* Bottom() { return (T*)stack_; } + const T* Top() const { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* End() { return reinterpret_cast(stackTop_); } + + template + const T* End() const { return reinterpret_cast(stackTop_); } + + template + T* Bottom() { return reinterpret_cast(stack_); } + + template + const T* Bottom() const { return reinterpret_cast(stack_); } + + bool HasAllocator() const { + return allocator_ != 0; + } + + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } - Allocator& GetAllocator() { return *allocator_; } bool Empty() const { return stackTop_ == stack_; } size_t GetSize() const { return static_cast(stackTop_ - stack_); } size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } @@ -136,7 +184,7 @@ private: size_t newCapacity; if (stack_ == 0) { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); newCapacity = initialCapacity_; } else { newCapacity = GetCapacity(); @@ -151,7 +199,7 @@ private: void Resize(size_t newCapacity) { const size_t size = GetSize(); // Backup the current size - stack_ = (char*)allocator_->Realloc(stack_, GetCapacity(), newCapacity); + stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); stackTop_ = stack_ + size; stackEnd_ = stack_ + newCapacity; } @@ -176,4 +224,8 @@ private: } // namespace internal RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_STACK_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/strfunc.h b/contrib/rapidjson/include/rapidjson/internal/strfunc.h index 84405065a..226439a76 100644 --- a/contrib/rapidjson/include/rapidjson/internal/strfunc.h +++ b/contrib/rapidjson/include/rapidjson/internal/strfunc.h @@ -15,7 +15,8 @@ #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ #define RAPIDJSON_INTERNAL_STRFUNC_H_ -#include "../rapidjson.h" +#include "../stream.h" +#include RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -28,11 +29,40 @@ namespace internal { */ template inline SizeType StrLen(const Ch* s) { + RAPIDJSON_ASSERT(s != 0); const Ch* p = s; while (*p) ++p; return SizeType(p - s); } +template <> +inline SizeType StrLen(const char* s) { + return SizeType(std::strlen(s)); +} + +template <> +inline SizeType StrLen(const wchar_t* s) { + return SizeType(std::wcslen(s)); +} + +//! Returns number of code points in a encoded string. +template +bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { + RAPIDJSON_ASSERT(s != 0); + RAPIDJSON_ASSERT(outCount != 0); + GenericStringStream is(s); + const typename Encoding::Ch* end = s + length; + SizeType count = 0; + while (is.src_ < end) { + unsigned codepoint; + if (!Encoding::Decode(is, &codepoint)) + return false; + count++; + } + *outCount = count; + return true; +} + } // namespace internal RAPIDJSON_NAMESPACE_END diff --git a/contrib/rapidjson/include/rapidjson/internal/strtod.h b/contrib/rapidjson/include/rapidjson/internal/strtod.h index ace65f677..adf49e349 100644 --- a/contrib/rapidjson/include/rapidjson/internal/strtod.h +++ b/contrib/rapidjson/include/rapidjson/internal/strtod.h @@ -15,7 +15,6 @@ #ifndef RAPIDJSON_STRTOD_ #define RAPIDJSON_STRTOD_ -#include "../rapidjson.h" #include "ieee754.h" #include "biginteger.h" #include "diyfp.h" @@ -95,13 +94,13 @@ inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { hS_Exp2 -= common_Exp2; BigInteger dS = d; - dS.MultiplyPow5(dS_Exp5) <<= dS_Exp2; + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); BigInteger bS(bInt); - bS.MultiplyPow5(bS_Exp5) <<= bS_Exp2; + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); BigInteger hS(1); - hS.MultiplyPow5(hS_Exp5) <<= hS_Exp2; + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); BigInteger delta(0); dS.Difference(bS, &delta); @@ -134,22 +133,22 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) break; - significand = significand * 10 + (decimals[i] - '0'); + significand = significand * 10u + static_cast(decimals[i] - '0'); } if (i < length && decimals[i] >= '5') // Rounding significand++; size_t remaining = length - i; - const unsigned kUlpShift = 3; - const unsigned kUlp = 1 << kUlpShift; - int error = (remaining == 0) ? 0 : kUlp / 2; + const int kUlpShift = 3; + const int kUlp = 1 << kUlpShift; + int64_t error = (remaining == 0) ? 0 : kUlp / 2; DiyFp v(significand, 0); v = v.Normalize(); error <<= -v.e; - const int dExp = (int)decimalPosition - (int)i + exp; + const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; int actualExp; DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); @@ -163,10 +162,10 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 }; - int adjustment = dExp - actualExp - 1; + int adjustment = dExp - actualExp - 1; RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); v = v * kPow10[adjustment]; - if (length + adjustment > 19) // has more digits than decimal digits in 64-bit + if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit error += kUlp / 2; } @@ -178,10 +177,10 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit v = v.Normalize(); error <<= oldExp - v.e; - const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); - unsigned precisionSize = 64 - effectiveSignificandSize; + const int effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + int precisionSize = 64 - effectiveSignificandSize; if (precisionSize + kUlpShift >= 64) { - unsigned scaleExp = (precisionSize + kUlpShift) - 63; + int scaleExp = (precisionSize + kUlpShift) - 63; v.f >>= scaleExp; v.e += scaleExp; error = (error >> scaleExp) + 1 + kUlp; @@ -191,7 +190,7 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit DiyFp rounded(v.f >> precisionSize, v.e + precisionSize); const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; - if (precisionBits >= halfWay + error) { + if (precisionBits >= halfWay + static_cast(error)) { rounded.f++; if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) rounded.f >>= 1; @@ -201,12 +200,12 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit *result = rounded.ToDouble(); - return halfWay - error >= precisionBits || precisionBits >= halfWay + error; + return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { const BigInteger dInt(decimals, length); - const int dExp = (int)decimalPosition - (int)length + exp; + const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; Double a(approx); int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); if (cmp < 0) @@ -246,10 +245,10 @@ inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t // Trim right-most digits const int kMaxDecimalDigit = 780; - if ((int)length > kMaxDecimalDigit) { - int delta = (int(length) - kMaxDecimalDigit); + if (static_cast(length) > kMaxDecimalDigit) { + int delta = (static_cast(length) - kMaxDecimalDigit); exp += delta; - decimalPosition -= delta; + decimalPosition -= static_cast(delta); length = kMaxDecimalDigit; } diff --git a/contrib/rapidjson/include/rapidjson/internal/swap.h b/contrib/rapidjson/include/rapidjson/internal/swap.h index 0590921f1..666e49f97 100644 --- a/contrib/rapidjson/include/rapidjson/internal/swap.h +++ b/contrib/rapidjson/include/rapidjson/internal/swap.h @@ -17,6 +17,11 @@ #include "../rapidjson.h" +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -34,4 +39,8 @@ inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { } // namespace internal RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/contrib/rapidjson/include/rapidjson/istreamwrapper.h b/contrib/rapidjson/include/rapidjson/istreamwrapper.h new file mode 100644 index 000000000..8639c8c3c --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/istreamwrapper.h @@ -0,0 +1,115 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ISTREAMWRAPPER_H_ +#define RAPIDJSON_ISTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::istringstream + - \c std::stringstream + - \c std::wistringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wifstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_istream. +*/ + +template +class BasicIStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} + + Ch Peek() const { + typename StreamType::int_type c = stream_.peek(); + return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : static_cast('\0'); + } + + Ch Take() { + typename StreamType::int_type c = stream_.get(); + if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { + count_++; + return static_cast(c); + } + else + return '\0'; + } + + // tellg() may return -1 when failed. So we count by ourself. + size_t Tell() const { return count_; } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. + int i; + bool hasError = false; + for (i = 0; i < 4; ++i) { + typename StreamType::int_type c = stream_.get(); + if (c == StreamType::traits_type::eof()) { + hasError = true; + stream_.clear(); + break; + } + peekBuffer_[i] = static_cast(c); + } + for (--i; i >= 0; --i) + stream_.putback(peekBuffer_[i]); + return !hasError ? peekBuffer_ : 0; + } + +private: + BasicIStreamWrapper(const BasicIStreamWrapper&); + BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); + + StreamType& stream_; + size_t count_; //!< Number of characters read. Note: + mutable Ch peekBuffer_[4]; +}; + +typedef BasicIStreamWrapper IStreamWrapper; +typedef BasicIStreamWrapper WIStreamWrapper; + +#if defined(__clang__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/contrib/rapidjson/include/rapidjson/memorybuffer.h b/contrib/rapidjson/include/rapidjson/memorybuffer.h index 2484b2185..39bee1dec 100644 --- a/contrib/rapidjson/include/rapidjson/memorybuffer.h +++ b/contrib/rapidjson/include/rapidjson/memorybuffer.h @@ -15,7 +15,7 @@ #ifndef RAPIDJSON_MEMORYBUFFER_H_ #define RAPIDJSON_MEMORYBUFFER_H_ -#include "rapidjson.h" +#include "stream.h" #include "internal/stack.h" RAPIDJSON_NAMESPACE_BEGIN diff --git a/contrib/rapidjson/include/rapidjson/memorystream.h b/contrib/rapidjson/include/rapidjson/memorystream.h index 99feae5d7..1d71d8a4f 100644 --- a/contrib/rapidjson/include/rapidjson/memorystream.h +++ b/contrib/rapidjson/include/rapidjson/memorystream.h @@ -15,7 +15,13 @@ #ifndef RAPIDJSON_MEMORYSTREAM_H_ #define RAPIDJSON_MEMORYSTREAM_H_ -#include "rapidjson.h" +#include "stream.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif RAPIDJSON_NAMESPACE_BEGIN @@ -36,8 +42,8 @@ struct MemoryStream { MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} - Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } - Ch Take() { return (src_ == end_) ? '\0' : *src_++; } + Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } + Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } size_t Tell() const { return static_cast(src_ - begin_); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } @@ -58,4 +64,8 @@ struct MemoryStream { RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h b/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h index a26fff4bf..3d4477b9a 100644 --- a/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h +++ b/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h @@ -89,14 +89,14 @@ #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we should wrap include with 'extern "C++" {}' -// or compiler give many errors like this: +// compiling for ARM we have to wrap include with 'extern "C++" {}' +// or compiler would give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#ifdef __cplusplus +#if defined(__cplusplus) && !defined(_M_ARM) extern "C" { #endif # include -#ifdef __cplusplus +#if defined(__cplusplus) && !defined(_M_ARM) } #endif diff --git a/contrib/rapidjson/include/rapidjson/ostreamwrapper.h b/contrib/rapidjson/include/rapidjson/ostreamwrapper.h new file mode 100644 index 000000000..6f4667c08 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/ostreamwrapper.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ +#define RAPIDJSON_OSTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::ostringstream + - \c std::stringstream + - \c std::wpstringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wofstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_ostream. +*/ + +template +class BasicOStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + + void Put(Ch c) { + stream_.put(c); + } + + void Flush() { + stream_.flush(); + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + + StreamType& stream_; +}; + +typedef BasicOStreamWrapper OStreamWrapper; +typedef BasicOStreamWrapper WOStreamWrapper; + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/contrib/rapidjson/include/rapidjson/pointer.h b/contrib/rapidjson/include/rapidjson/pointer.h index 5d2aa8d63..0f377efec 100644 --- a/contrib/rapidjson/include/rapidjson/pointer.h +++ b/contrib/rapidjson/include/rapidjson/pointer.h @@ -18,6 +18,16 @@ #include "document.h" #include "internal/itoa.h" +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + RAPIDJSON_NAMESPACE_BEGIN static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token @@ -71,7 +81,7 @@ template class GenericPointer { public: typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value - typedef typename EncodingType::Ch Ch; //!< Character type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value //! A token is the basic units of internal representation. /*! @@ -96,7 +106,7 @@ public: //@{ //! Default constructor. - GenericPointer() : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Constructor that parses a string or URI fragment representation. /*! @@ -155,7 +165,7 @@ public: GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Copy constructor. - GenericPointer(const GenericPointer& rhs) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { *this = rhs; } @@ -230,7 +240,7 @@ public: template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) Append(T* name, Allocator* allocator = 0) const { - return Append(name, StrLen(name), allocator); + return Append(name, internal::StrLen(name), allocator); } #if RAPIDJSON_HAS_STDSTRING @@ -253,17 +263,18 @@ public: */ GenericPointer Append(SizeType index, Allocator* allocator = 0) const { char buffer[21]; - SizeType length = (sizeof(SizeType) == 4 ? internal::u32toa(index, buffer): internal::u64toa(index, buffer)) - buffer; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); buffer[length] = '\0'; if (sizeof(Ch) == 1) { - Token token = { (Ch*)buffer, length, index }; + Token token = { reinterpret_cast(buffer), length, index }; return Append(token, allocator); } else { Ch name[21]; for (size_t i = 0; i <= length; i++) - name[i] = buffer[i]; + name[i] = static_cast(buffer[i]); Token token = { name, length, index }; return Append(token, allocator); } @@ -271,7 +282,7 @@ public: //! Append a token by value, and return a new Pointer /*! - \param value Value (either Uint or String) to be appended. + \param token token to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ @@ -299,6 +310,9 @@ public: //@} + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } + //!@name Tokens //@{ @@ -390,7 +404,7 @@ public: bool exist = true; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { if (v->IsArray() && t->name[0] == '-' && t->length == 1) { - v->PushBack(Value().Move(), allocator); + v->PushBack(ValueType().Move(), allocator); v = &((*v)[v->Size() - 1]); exist = false; } @@ -408,7 +422,7 @@ public: if (t->index >= v->Size()) { v->Reserve(t->index + 1, allocator); while (t->index >= v->Size()) - v->PushBack(Value().Move(), allocator); + v->PushBack(ValueType().Move(), allocator); exist = false; } v = &((*v)[t->index]); @@ -416,7 +430,7 @@ public: else { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) { - v->AddMember(Value(t->name, t->length, allocator).Move(), Value().Move(), allocator); + v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end exist = false; } @@ -435,7 +449,6 @@ public: //! Creates a value in a document. /*! \param document A document to be resolved. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. \param alreadyExist If non-null, it stores whether the resolved value is already exist. \return The resolved newly created, or already exists value. */ @@ -452,9 +465,18 @@ public: //! Query a value in a subtree. /*! \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. */ - ValueType* Get(ValueType& root) const { + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { RAPIDJSON_ASSERT(IsValid()); ValueType* v = &root; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { @@ -463,18 +485,23 @@ public: { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) - return 0; + break; v = &m->value; } - break; + continue; case kArrayType: if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - return 0; + break; v = &((*v)[t->index]); - break; + continue; default: - return 0; + break; } + + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; } return v; } @@ -484,7 +511,9 @@ public: \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \return Pointer to the value if it can be resolved. Otherwise null. */ - const ValueType* Get(const ValueType& root) const { return Get(const_cast(root)); } + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { + return Get(const_cast(root), unresolvedTokenIndex); + } //@} @@ -525,7 +554,7 @@ public: //! Query a value in a subtree with default primitive value. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -555,7 +584,7 @@ public: //! Query a value in a document with default primitive value. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -601,7 +630,7 @@ public: //! Set a primitive value in a subtree. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -637,7 +666,7 @@ public: //! Set a primitive value in a document. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -729,7 +758,7 @@ private: */ Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) @@ -738,8 +767,12 @@ private: tokenCount_ = rhs.tokenCount_ + extraToken; tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); - std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + if (rhs.tokenCount_ > 0) { + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + } + if (nameBufferSize > 0) { + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + } // Adjust pointers to name buffer std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; @@ -759,11 +792,13 @@ private: } //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation /*! \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. \param length Length of the source string. \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. */ +#endif void Parse(const Ch* source, size_t length) { RAPIDJSON_ASSERT(source != NULL); RAPIDJSON_ASSERT(nameBuffer_ == 0); @@ -771,7 +806,7 @@ private: // Create own allocator if user did not supply. if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); // Count number of '/' as tokenCount tokenCount_ = 0; @@ -857,7 +892,7 @@ private: *name++ = c; } - token->length = name - token->name; + token->length = static_cast(name - token->name); if (token->length == 0) isNumber = false; *name++ = '\0'; // Null terminator @@ -944,6 +979,8 @@ private: */ class PercentDecodeStream { public: + typedef typename ValueType::Ch Ch; + //! Constructor /*! \param source Start of the stream @@ -959,11 +996,11 @@ private: src_++; Ch c = 0; for (int j = 0; j < 2; j++) { - c <<= 4; + c = static_cast(c << 4); Ch h = *src_; - if (h >= '0' && h <= '9') c += h - '0'; - else if (h >= 'A' && h <= 'F') c += h - 'A' + 10; - else if (h >= 'a' && h <= 'f') c += h - 'a' + 10; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); else { valid_ = false; return 0; @@ -973,7 +1010,7 @@ private: return c; } - size_t Tell() const { return src_ - head_; } + size_t Tell() const { return static_cast(src_ - head_); } bool IsValid() const { return valid_; } private: @@ -992,8 +1029,8 @@ private: unsigned char u = static_cast(c); static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; os_.Put('%'); - os_.Put(hexDigits[u >> 4]); - os_.Put(hexDigits[u & 15]); + os_.Put(static_cast(hexDigits[u >> 4])); + os_.Put(static_cast(hexDigits[u & 15])); } private: OutputStream& os_; @@ -1041,23 +1078,23 @@ typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, c ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer) { - return pointer.Get(root); +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); } template -const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer) { - return pointer.Get(root); +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); } template -typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N]) { - return GenericPointer(source, N - 1).Get(root); +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } template -const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N]) { - return GenericPointer(source, N - 1).Get(root); +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } ////////////////////////////////////////////////////////////////////////////// @@ -1310,4 +1347,12 @@ bool EraseValueByPointer(T& root, const CharType(&source)[N]) { RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_POINTER_H_ diff --git a/contrib/rapidjson/include/rapidjson/prettywriter.h b/contrib/rapidjson/include/rapidjson/prettywriter.h index 416dd492e..98dfb3060 100644 --- a/contrib/rapidjson/include/rapidjson/prettywriter.h +++ b/contrib/rapidjson/include/rapidjson/prettywriter.h @@ -22,8 +22,21 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN +//! Combination of PrettyWriter format flags. +/*! \see PrettyWriter::SetFormatOptions + */ +enum PrettyFormatOptions { + kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. +}; + //! Writer with indentation and spacing. /*! \tparam OutputStream Type of ouptut os. @@ -31,10 +44,10 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam TargetEncoding Encoding of output stream. \tparam StackAllocator Type of allocator for allocating memory of stack. */ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> -class PrettyWriter : public Writer { +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class PrettyWriter : public Writer { public: - typedef Writer Base; + typedef Writer Base; typedef typename Base::Ch Ch; //! Constructor @@ -42,8 +55,17 @@ public: \param allocator User supplied allocator. If it is null, it will create a private one. \param levelDepth Initial capacity of stack. */ - PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + PrettyWriter(PrettyWriter&& rhs) : + Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} +#endif //! Set custom indentation. /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). @@ -57,6 +79,14 @@ public: return *this; } + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { + formatOptions_ = options; + return *this; + } + /*! @name Implementation of Handler \see Handler */ @@ -70,7 +100,15 @@ public: bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); + (void)copy; + PrettyPrefix(kNumberType); + return Base::WriteString(str, length); + } + bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; PrettyPrefix(kStringType); return Base::WriteString(str, length); @@ -89,11 +127,19 @@ public: } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) { + return Key(str.data(), SizeType(str.size())); + } +#endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; if (!empty) { @@ -104,7 +150,7 @@ public: (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); + Base::Flush(); return true; } @@ -120,7 +166,7 @@ public: RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - if (!empty) { + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); WriteIndent(); } @@ -128,7 +174,7 @@ public: (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); + Base::Flush(); return true; } @@ -142,6 +188,22 @@ public: bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + PrettyPrefix(type); + return Base::WriteRawValue(json, length); + } + protected: void PrettyPrefix(Type type) { (void)type; @@ -151,11 +213,14 @@ protected: if (level->inArray) { if (level->valueCount > 0) { Base::os_->Put(','); // add comma if it is not the first element in array - Base::os_->Put('\n'); + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); } - else + + if (!(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); - WriteIndent(); + WriteIndent(); + } } else { // in object if (level->valueCount > 0) { @@ -186,11 +251,12 @@ protected: void WriteIndent() { size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, indentChar_, count); + PutN(*Base::os_, static_cast(indentChar_), count); } Ch indentChar_; unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; private: // Prohibit copy constructor & assignment operator. @@ -200,6 +266,10 @@ private: RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/rapidjson.h b/contrib/rapidjson/include/rapidjson/rapidjson.h index f22130d3c..5716fdc06 100644 --- a/contrib/rapidjson/include/rapidjson/rapidjson.h +++ b/contrib/rapidjson/include/rapidjson/rapidjson.h @@ -49,6 +49,11 @@ // token stringification #define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) #define RAPIDJSON_DO_STRINGIFY(x) #x + +// token concatenation +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y //!@endcond /*! \def RAPIDJSON_MAJOR_VERSION @@ -68,8 +73,8 @@ \brief Version of RapidJSON in ".." string format. */ #define RAPIDJSON_MAJOR_VERSION 1 -#define RAPIDJSON_MINOR_VERSION 0 -#define RAPIDJSON_PATCH_VERSION 2 +#define RAPIDJSON_MINOR_VERSION 1 +#define RAPIDJSON_PATCH_VERSION 0 #define RAPIDJSON_VERSION_STRING \ RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) @@ -119,6 +124,31 @@ #define RAPIDJSON_NAMESPACE_END } #endif +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_HAS_STDSTRING +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def RAPIDJSON_HAS_STDSTRING + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#endif // !defined(RAPIDJSON_HAS_STDSTRING) + +#if RAPIDJSON_HAS_STDSTRING +#include +#endif // RAPIDJSON_HAS_STDSTRING + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE @@ -134,7 +164,7 @@ */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#ifdef _MSC_VER +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else @@ -153,9 +183,9 @@ #ifndef RAPIDJSON_FORCEINLINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && !defined(NDEBUG) +#if defined(_MSC_VER) && defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __forceinline -#elif defined(__GNUC__) && __GNUC__ >= 4 && !defined(NDEBUG) +#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) #else #define RAPIDJSON_FORCEINLINE @@ -211,6 +241,8 @@ # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else @@ -223,7 +255,7 @@ //! Whether using 64-bit architecture #ifndef RAPIDJSON_64BIT -#if defined(__LP64__) || defined(_WIN64) +#if defined(__LP64__) || (defined(__x86_64__) && defined(__ILP32__)) || defined(_WIN64) || defined(__EMSCRIPTEN__) #define RAPIDJSON_64BIT 1 #else #define RAPIDJSON_64BIT 0 @@ -238,13 +270,14 @@ \param x pointer to align Some machines require strict data alignment. Currently the default uses 4 bytes - alignment. User can customize by defining the RAPIDJSON_ALIGN function macro., + alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. + User can customize by defining the RAPIDJSON_ALIGN function macro. */ #ifndef RAPIDJSON_ALIGN #if RAPIDJSON_64BIT == 1 -#define RAPIDJSON_ALIGN(x) ((x + 7u) & ~7u) +#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) #else -#define RAPIDJSON_ALIGN(x) ((x + 3u) & ~3u) +#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) #endif #endif @@ -262,17 +295,47 @@ #endif /////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD +// RAPIDJSON_48BITPOINTER_OPTIMIZATION + +//! Use only lower 48-bit address for some pointers. +/*! + \ingroup RAPIDJSON_CONFIG + + This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. + The higher 16-bit can be used for storing other data. + \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. +*/ +#ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 +#else +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 +#endif +#endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION + +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 +#if RAPIDJSON_64BIT != 1 +#error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 +#endif +#define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) +#define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) +#else +#define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) +#define RAPIDJSON_GETPOINTER(type, p) (p) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_NEON/RAPIDJSON_SIMD /*! \def RAPIDJSON_SIMD \ingroup RAPIDJSON_CONFIG - \brief Enable SSE2/SSE4.2 optimization. + \brief Enable SSE2/SSE4.2/Neon optimization. RapidJSON supports optimized implementations for some parsing operations - based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible - processors. + based on the SSE2, SSE4.2 or NEon SIMD extensions on modern Intel + or ARM compatible processors. - To enable these optimizations, two different symbols can be defined; + To enable these optimizations, three different symbols can be defined; \code // Enable SSE2 optimization. #define RAPIDJSON_SSE2 @@ -281,13 +344,17 @@ #define RAPIDJSON_SSE42 \endcode - \c RAPIDJSON_SSE42 takes precedence, if both are defined. + // Enable ARM Neon optimization. + #define RAPIDJSON_NEON + \endcode + + \c RAPIDJSON_SSE42 takes precedence over SSE2, if both are defined. If any of these symbols is defined, RapidJSON defines the macro \c RAPIDJSON_SIMD to indicate the availability of the optimized code. */ #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ - || defined(RAPIDJSON_DOXYGEN_RUNNING) + || defined(RAPIDJSON_NEON) || defined(RAPIDJSON_DOXYGEN_RUNNING) #define RAPIDJSON_SIMD #endif @@ -347,25 +414,33 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_STATIC_ASSERT -// Adopt from boost +// Prefer C++11 static_assert, if available #ifndef RAPIDJSON_STATIC_ASSERT +#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#define RAPIDJSON_STATIC_ASSERT(x) \ + static_assert(x, RAPIDJSON_STRINGIFY(x)) +#endif // C++11 +#endif // RAPIDJSON_STATIC_ASSERT + +// Adopt C++03 implementation from boost +#ifndef RAPIDJSON_STATIC_ASSERT +#ifndef __clang__ //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#endif RAPIDJSON_NAMESPACE_BEGIN template struct STATIC_ASSERTION_FAILURE; template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; -template struct StaticAssertTest {}; +template struct StaticAssertTest {}; RAPIDJSON_NAMESPACE_END -#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) -#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) -#define RAPIDJSON_DO_JOIN2(X, Y) X##Y - #if defined(__GNUC__) #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) #else #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif +#ifndef __clang__ //!@endcond +#endif /*! \def RAPIDJSON_STATIC_ASSERT \brief (Internal) macro to check for conditions at compile-time @@ -376,6 +451,35 @@ RAPIDJSON_NAMESPACE_END typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif // RAPIDJSON_STATIC_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY + +//! Compiler branching hint for expression with high probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression likely to be true. +*/ +#ifndef RAPIDJSON_LIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define RAPIDJSON_LIKELY(x) (x) +#endif +#endif + +//! Compiler branching hint for expression with low probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression unlikely to be true. +*/ +#ifndef RAPIDJSON_UNLIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define RAPIDJSON_UNLIKELY(x) (x) +#endif #endif /////////////////////////////////////////////////////////////////////////////// @@ -438,8 +542,12 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS #if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS __has_feature(cxx_rvalue_references) && \ +#if __has_feature(cxx_rvalue_references) && \ (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1600) @@ -470,6 +578,17 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 #endif +#ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#else +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RANGE_FOR + //!@endcond /////////////////////////////////////////////////////////////////////////////// @@ -477,7 +596,7 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_NEW ///! customization point for global \c new -#define RAPIDJSON_NEW(x) new x +#define RAPIDJSON_NEW(TypeName) new TypeName #endif #ifndef RAPIDJSON_DELETE ///! customization point for global \c delete @@ -485,10 +604,7 @@ RAPIDJSON_NAMESPACE_END #endif /////////////////////////////////////////////////////////////////////////////// -// Allocators and Encodings - -#include "allocators.h" -#include "encodings.h" +// Type /*! \namespace rapidjson \brief main RapidJSON namespace @@ -496,148 +612,6 @@ RAPIDJSON_NAMESPACE_END */ RAPIDJSON_NAMESPACE_BEGIN -/////////////////////////////////////////////////////////////////////////////// -// Stream - -/*! \class rapidjson::Stream - \brief Concept for reading and writing characters. - - For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). - - For write-only stream, only need to implement Put() and Flush(). - -\code -concept Stream { - typename Ch; //!< Character type of the stream. - - //! Read the current character from stream without moving the read cursor. - Ch Peek() const; - - //! Read the current character from stream and moving the read cursor to next character. - Ch Take(); - - //! Get the current read cursor. - //! \return Number of characters read from start. - size_t Tell(); - - //! Begin writing operation at the current read pointer. - //! \return The begin writer pointer. - Ch* PutBegin(); - - //! Write a character. - void Put(Ch c); - - //! Flush the buffer. - void Flush(); - - //! End the writing operation. - //! \param begin The begin write pointer returned by PutBegin(). - //! \return Number of characters written. - size_t PutEnd(Ch* begin); -} -\endcode -*/ - -//! Provides additional information for stream. -/*! - By using traits pattern, this type provides a default configuration for stream. - For custom stream, this type can be specialized for other configuration. - See TEST(Reader, CustomStringStream) in readertest.cpp for example. -*/ -template -struct StreamTraits { - //! Whether to make local copy of stream for optimization during parsing. - /*! - By default, for safety, streams do not use local copy optimization. - Stream that can be copied fast should specialize this, like StreamTraits. - */ - enum { copyOptimization = 0 }; -}; - -//! Put N copies of a character to a stream. -template -inline void PutN(Stream& stream, Ch c, size_t n) { - for (size_t i = 0; i < n; i++) - stream.Put(c); -} - -/////////////////////////////////////////////////////////////////////////////// -// StringStream - -//! Read-only string stream. -/*! \note implements Stream concept -*/ -template -struct GenericStringStream { - typedef typename Encoding::Ch Ch; - - GenericStringStream(const Ch *src) : src_(src), head_(src) {} - - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast(src_ - head_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! String stream with UTF8 encoding. -typedef GenericStringStream > StringStream; - -/////////////////////////////////////////////////////////////////////////////// -// InsituStringStream - -//! A read-write string stream. -/*! This string stream is particularly designed for in-situ parsing. - \note implements Stream concept -*/ -template -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; - - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return static_cast(src_ - head_); } - - // Write - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - - Ch* PutBegin() { return dst_ = src_; } - size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } - void Flush() {} - - Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } - void Pop(size_t count) { dst_ -= count; } - - Ch* src_; - Ch* dst_; - Ch* head_; -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! Insitu string stream with UTF8 encoding. -typedef GenericInsituStringStream > InsituStringStream; - -/////////////////////////////////////////////////////////////////////////////// -// Type - //! Type of JSON value enum Type { kNullType = 0, //!< null diff --git a/contrib/rapidjson/include/rapidjson/reader.h b/contrib/rapidjson/include/rapidjson/reader.h index c5ecf4be5..120c31115 100644 --- a/contrib/rapidjson/include/rapidjson/reader.h +++ b/contrib/rapidjson/include/rapidjson/reader.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_READER_H_ @@ -17,11 +17,13 @@ /*! \file reader.h */ -#include "rapidjson.h" -#include "encodings.h" +#include "allocators.h" +#include "stream.h" +#include "encodedstream.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strtod.h" +#include #if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) #include @@ -31,6 +33,8 @@ #include #elif defined(RAPIDJSON_SSE2) #include +#elif defined(RAPIDJSON_NEON) +#include #endif #ifdef _MSC_VER @@ -39,6 +43,13 @@ RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant RAPIDJSON_DIAG_OFF(4702) // unreachable code #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(old-style-cast) +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) @@ -49,7 +60,7 @@ RAPIDJSON_DIAG_OFF(effc++) #ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ - if (HasParseError()) { return value; } \ + if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ RAPIDJSON_MULTILINEMACRO_END #endif #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ @@ -120,7 +131,7 @@ RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // ParseFlag -/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS +/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS \ingroup RAPIDJSON_CONFIG \brief User-defined kParseDefaultFlags definition. @@ -140,6 +151,10 @@ enum ParseFlag { kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. + kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. + kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; @@ -148,7 +163,7 @@ enum ParseFlag { /*! \class rapidjson::Handler \brief Concept for receiving events from GenericReader upon parsing. - The functions return true if no error occurs. If they return false, + The functions return true if no error occurs. If they return false, the event publisher should terminate the process. \code concept Handler { @@ -161,6 +176,8 @@ concept Handler { bool Int64(int64_t i); bool Uint64(uint64_t i); bool Double(double d); + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType length, bool copy); bool String(const Ch* str, SizeType length, bool copy); bool StartObject(); bool Key(const Ch* str, SizeType length, bool copy); @@ -191,6 +208,8 @@ struct BaseReaderHandler { bool Int64(int64_t) { return static_cast(*this).Default(); } bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } @@ -248,10 +267,17 @@ void SkipWhitespace(InputStream& is) { internal::StreamLocalCopy copy(is); InputStream& s(copy.s); - while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') + typename InputStream::Ch c; + while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') s.Take(); } +inline const char* SkipWhitespace(const char* p, const char* end) { + while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + return p; +} + #ifdef RAPIDJSON_SSE42 //! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. inline const char *SkipWhitespace_SIMD(const char* p) { @@ -262,7 +288,7 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; @@ -271,23 +297,37 @@ inline const char *SkipWhitespace_SIMD(const char* p) { // The rest of string using SIMD static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_load_si128((const __m128i *)&whitespace[0]); + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); for (;; p += 16) { - const __m128i s = _mm_load_si128((const __m128i *)p); - const unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; } } +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The middle of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; + } + + return SkipWhitespace(p, end); +} + #elif defined(RAPIDJSON_SSE2) //! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. @@ -299,7 +339,7 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; @@ -307,24 +347,22 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // The rest of string - static const char whitespaces[4][17] = { - " ", - "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", - "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r", - "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"}; + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 - const __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]); - const __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]); - const __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]); - const __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]); + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); for (;; p += 16) { - const __m128i s = _mm_load_si128((const __m128i *)p); + const __m128i s = _mm_load_si128(reinterpret_cast(p)); __m128i x = _mm_cmpeq_epi8(s, w0); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = (unsigned short)~_mm_movemask_epi8(x); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); if (r != 0) { // some of characters may be non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; @@ -337,11 +375,134 @@ inline const char *SkipWhitespace_SIMD(const char* p) { } } -#endif // RAPIDJSON_SSE2 +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#elif defined(RAPIDJSON_NEON) + +//! Skip whitespace with ARM Neon instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz =__builtin_clzll(high);; + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low);; + return p + (lz >> 3); + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (; p <= end - 16; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low); + return p + (lz >> 3); + } + } + + return SkipWhitespace(p, end); +} + +#endif // RAPIDJSON_NEON #ifdef RAPIDJSON_SIMD //! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& is) { +template<> inline void SkipWhitespace(InsituStringStream& is) { is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); } @@ -349,23 +510,27 @@ template<> inline void SkipWhitespace(InsituStringStream& is) { template<> inline void SkipWhitespace(StringStream& is) { is.src_ = SkipWhitespace_SIMD(is.src_); } + +template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { + is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); +} #endif // RAPIDJSON_SIMD /////////////////////////////////////////////////////////////////////////////// // GenericReader //! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. -/*! GenericReader parses JSON text from a stream, and send events synchronously to an +/*! GenericReader parses JSON text from a stream, and send events synchronously to an object implementing Handler concept. - It needs to allocate a stack for storing a single decoded string during + It needs to allocate a stack for storing a single decoded string during non-destructive parsing. - For in-situ parsing, the decoded string is directly written to the source + For in-situ parsing, the decoded string is directly written to the source text string, no temporary buffer is required. A GenericReader object can be reused for parsing multiple JSON text. - + \tparam SourceEncoding Encoding of the input stream. \tparam TargetEncoding Encoding of the parse output. \tparam StackAllocator Allocator type for stack. @@ -398,9 +563,10 @@ public: ClearStackOnExit scope(*this); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - if (is.Peek() == '\0') { + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } @@ -409,9 +575,10 @@ public: RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - if (is.Peek() != '\0') { + if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } @@ -433,9 +600,86 @@ public: return Parse(is, handler); } + //! Initialize JSON text token-by-token parsing + /*! + */ + void IterativeParseInit() { + parseResult_.Clear(); + state_ = IterativeParsingStartState; + } + + //! Parse one token from JSON text + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + bool IterativeParseNext(InputStream& is, Handler& handler) { + while (RAPIDJSON_LIKELY(is.Peek() != '\0')) { + SkipWhitespaceAndComments(is); + + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state_, t); + IterativeParsingState d = Transit(state_, t, n, is, handler); + + // If we've finished or hit an error... + if (RAPIDJSON_UNLIKELY(IsIterativeParsingCompleteState(d))) { + // Report errors. + if (d == IterativeParsingErrorState) { + HandleError(state_, is); + return false; + } + + // Transition to the finish state. + RAPIDJSON_ASSERT(d == IterativeParsingFinishState); + state_ = d; + + // If StopWhenDone is not set... + if (!(parseFlags & kParseStopWhenDoneFlag)) { + // ... and extra non-whitespace data is found... + SkipWhitespaceAndComments(is); + if (is.Peek() != '\0') { + // ... this is considered an error. + HandleError(state_, is); + return false; + } + } + + // Success! We are done! + return true; + } + + // Transition to the new state. + state_ = d; + + // If we parsed anything other than a delimiter, we invoked the handler, so we can return true now. + if (!IsIterativeParsingDelimiterState(n)) + return true; + } + + // We reached the end of file. + stack_.Clear(); + + if (state_ != IterativeParsingFinishState) { + HandleError(state_, is); + return false; + } + + return true; + } + + //! Check if token-by-token parsing JSON text is complete + /*! \return Whether the JSON has been fully decoded. + */ + RAPIDJSON_FORCEINLINE bool IterativeParseComplete() { + return IsIterativeParsingCompleteState(state_); + } + //! Whether a parse error has occured in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } - + //! Get the \ref ParseErrorCode of last parsing. ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } @@ -462,52 +706,98 @@ private: ClearStackOnExit& operator=(const ClearStackOnExit&); }; + template + void SkipWhitespaceAndComments(InputStream& is) { + SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) { + while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { + if (Consume(is, '*')) { + while (true) { + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + else if (Consume(is, '*')) { + if (Consume(is, '/')) + break; + } + else + is.Take(); + } + } + else if (RAPIDJSON_LIKELY(Consume(is, '/'))) + while (is.Peek() != '\0' && is.Take() != '\n') {} + else + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + SkipWhitespace(is); + } + } + } + // Parse object: { string : value, ... } template void ParseObject(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '{'); is.Take(); // Skip '{' - - if (!handler.StartObject()) + + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (is.Peek() == '}') { - is.Take(); - if (!handler.EndObject(0)) // empty object + if (Consume(is, '}')) { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } for (SizeType memberCount = 0;;) { - if (is.Peek() != '"') + if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); ParseString(is, handler, true); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (is.Take() != ':') + if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++memberCount; - switch (is.Take()) { - case ',': SkipWhitespace(is); break; - case '}': - if (!handler.EndObject(memberCount)) + switch (is.Peek()) { + case ',': + is.Take(); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; + case '}': + is.Take(); + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); + default: + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy + } + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == '}') { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } } } } @@ -517,15 +807,15 @@ private: void ParseArray(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '['); is.Take(); // Skip '[' - - if (!handler.StartArray()) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - - SkipWhitespace(is); - if (is.Peek() == ']') { - is.Take(); - if (!handler.EndArray(0)) // empty array + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } @@ -535,15 +825,28 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++elementCount; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - switch (is.Take()) { - case ',': SkipWhitespace(is); break; - case ']': - if (!handler.EndArray(elementCount)) + if (Consume(is, ',')) { + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + } + else if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == ']') { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + } } } } @@ -553,12 +856,12 @@ private: RAPIDJSON_ASSERT(is.Peek() == 'n'); is.Take(); - if (is.Take() == 'u' && is.Take() == 'l' && is.Take() == 'l') { - if (!handler.Null()) + if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { + if (RAPIDJSON_UNLIKELY(!handler.Null())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template @@ -566,12 +869,12 @@ private: RAPIDJSON_ASSERT(is.Peek() == 't'); is.Take(); - if (is.Take() == 'r' && is.Take() == 'u' && is.Take() == 'e') { - if (!handler.Bool(true)) + if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template @@ -579,20 +882,30 @@ private: RAPIDJSON_ASSERT(is.Peek() == 'f'); is.Take(); - if (is.Take() == 'a' && is.Take() == 'l' && is.Take() == 's' && is.Take() == 'e') { - if (!handler.Bool(false)) + if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { + if (RAPIDJSON_LIKELY(is.Peek() == expect)) { + is.Take(); + return true; + } + else + return false; } // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). template - unsigned ParseHex4(InputStream& is) { + unsigned ParseHex4(InputStream& is, size_t escapeOffset) { unsigned codepoint = 0; for (int i = 0; i < 4; i++) { - Ch c = is.Take(); + Ch c = is.Peek(); codepoint <<= 4; codepoint += static_cast(c); if (c >= '0' && c <= '9') @@ -602,9 +915,10 @@ private: else if (c >= 'a' && c <= 'f') codepoint -= 'a' - 10; else { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); } + is.Take(); } return codepoint; } @@ -619,7 +933,14 @@ private: *stack_.template Push() = c; ++length_; } + + RAPIDJSON_FORCEINLINE void* Push(SizeType count) { + length_ += count; + return stack_.template Push(count); + } + size_t Length() const { return length_; } + Ch* Pop() { return stack_.template Pop(length_); } @@ -638,6 +959,9 @@ private: internal::StreamLocalCopy copy(is); InputStream& s(copy.s); + RAPIDJSON_ASSERT(s.Peek() == '\"'); + s.Take(); // Skip '\"' + bool success = false; if (parseFlags & kParseInsituFlag) { typename InputStream::Ch *head = s.PutBegin(); @@ -645,7 +969,7 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; size_t length = s.PutEnd(head) - 1; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - const typename TargetEncoding::Ch* const str = (typename TargetEncoding::Ch*)head; + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); } else { @@ -656,7 +980,7 @@ private: const typename TargetEncoding::Ch* const str = stackStream.Pop(); success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); } - if (!success) + if (RAPIDJSON_UNLIKELY(!success)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); } @@ -667,74 +991,421 @@ private: //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 }; #undef Z16 //!@endcond - RAPIDJSON_ASSERT(is.Peek() == '\"'); - is.Take(); // Skip '\"' - for (;;) { + // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + Ch c = is.Peek(); - if (c == '\\') { // Escape + if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape + size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset is.Take(); - Ch e = is.Take(); - if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { - os.Put(escape[(unsigned char)e]); + Ch e = is.Peek(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { + is.Take(); + os.Put(static_cast(escape[static_cast(e)])); } - else if (e == 'u') { // Unicode - unsigned codepoint = ParseHex4(is); + else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode + is.Take(); + unsigned codepoint = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { // Handle UTF-16 surrogate pair - if (is.Take() != '\\' || is.Take() != 'u') - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); - unsigned codepoint2 = ParseHex4(is); + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; } TEncoding::Encode(os, codepoint); } else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); } - else if (c == '"') { // Closing double quote + else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote is.Take(); os.Put('\0'); // null-terminate the string return; } - else if (c == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); - else if ((unsigned)c < 0x20) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); - else { - if (parseFlags & kParseValidateEncodingFlag ? - !Transcoder::Validate(is, os) : - !Transcoder::Transcode(is, os)) + else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); + else RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); } + else { + size_t offset = is.Tell(); + if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); + } } } - template + template + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { + // Do nothing for generic version + } + +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType length; + #ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; + #else + length = static_cast(__builtin_ffs(r) - 1); + #endif + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16, q += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + for (const char* pend = p + length; p != pend; ) + *q++ = *p++; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + p += length; + break; + } + } + + is.src_ = is.dst_ = p; + } +#elif defined(RAPIDJSON_NEON) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high);; + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low);; + length = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + vst1q_u8(reinterpret_cast(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16, q += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + length = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + for (const char* pend = p + length; p != pend; ) { + *q++ = *p++; + } + break; + } + vst1q_u8(reinterpret_cast(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + p += 8 + (lz >> 3); + break; + } + } else { + int lz = __builtin_clzll(low); + p += lz >> 3; + break; + } + } + + is.src_ = is.dst_ = p; + } +#endif // RAPIDJSON_NEON + + template class NumberStream; template - class NumberStream { + class NumberStream { public: + typedef typename InputStream::Ch Ch; + NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } - ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char) {} + size_t Tell() { return is.Tell(); } size_t Length() { return 0; } const char* Pop() { return 0; } @@ -746,17 +1417,20 @@ private: }; template - class NumberStream : public NumberStream { - typedef NumberStream Base; + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : NumberStream(reader, is), stackStream(reader.stack_) {} - ~NumberStream() {} + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put((char)Base::is.Peek()); + stackStream.Put(static_cast(Base::is.Peek())); return Base::is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char c) { + stackStream.Put(c); + } + size_t Length() { return stackStream.Length(); } const char* Pop() { @@ -768,34 +1442,48 @@ private: StackStream stackStream; }; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + + RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } + }; + template void ParseNumber(InputStream& is, Handler& handler) { internal::StreamLocalCopy copy(is); - NumberStream s(*this, copy.s); + NumberStream s(*this, copy.s); + + size_t startOffset = s.Tell(); + double d = 0.0; + bool useNanOrInf = false; // Parse minus - bool minus = false; - if (s.Peek() == '-') { - minus = true; - s.Take(); - } + bool minus = Consume(s, '-'); // Parse int: zero / ( digit1-9 *DIGIT ) unsigned i = 0; uint64_t i64 = 0; bool use64bit = false; int significandDigit = 0; - if (s.Peek() == '0') { + if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { i = 0; s.TakePush(); } - else if (s.Peek() >= '1' && s.Peek() <= '9') { + else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { i = static_cast(s.TakePush() - '0'); if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 214748364) { // 2^31 = 2147483648 - if (i != 214748364 || s.Peek() > '8') { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 + if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { i64 = i; use64bit = true; break; @@ -805,9 +1493,9 @@ private: significandDigit++; } else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 429496729) { // 2^32 - 1 = 4294967295 - if (i != 429496729 || s.Peek() > '5') { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 + if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { i64 = i; use64bit = true; break; @@ -817,18 +1505,41 @@ private: significandDigit++; } } + // Parse NaN or Infinity here + else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { + if (Consume(s, 'N')) { + if (Consume(s, 'a') && Consume(s, 'N')) { + d = std::numeric_limits::quiet_NaN(); + useNanOrInf = true; + } + } + else if (RAPIDJSON_LIKELY(Consume(s, 'I'))) { + if (Consume(s, 'n') && Consume(s, 'f')) { + d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + useNanOrInf = true; + + if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) { + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } + } + + if (RAPIDJSON_UNLIKELY(!useNanOrInf)) { + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); // Parse 64bit int bool useDouble = false; - double d = 0.0; if (use64bit) { - if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC)) // 2^63 = 9223372036854775808 - if (i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8') { - d = i64; + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { + d = static_cast(i64); useDouble = true; break; } @@ -836,10 +1547,10 @@ private: significandDigit++; } else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) // 2^64 - 1 = 18446744073709551615 - if (i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5') { - d = i64; + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { + d = static_cast(i64); useDouble = true; break; } @@ -850,9 +1561,9 @@ private: // Force double for big integer if (useDouble) { - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (d >= 1.7976931348623157e307) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); d = d * 10 + (s.TakePush() - '0'); } } @@ -860,11 +1571,10 @@ private: // Parse frac = decimal-point 1*DIGIT int expFrac = 0; size_t decimalPosition; - if (s.Peek() == '.') { - s.Take(); + if (Consume(s, '.')) { decimalPosition = s.Length(); - if (!(s.Peek() >= '0' && s.Peek() <= '9')) + if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); if (!useDouble) { @@ -872,8 +1582,8 @@ private: // Use i64 to store significand in 64-bit architecture if (!use64bit) i64 = i; - - while (s.Peek() >= '0' && s.Peek() <= '9') { + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path break; else { @@ -884,19 +1594,19 @@ private: } } - d = (double)i64; + d = static_cast(i64); #else // Use double to store significand in 32-bit architecture - d = use64bit ? (double)i64 : (double)i; + d = static_cast(use64bit ? i64 : i); #endif useDouble = true; } - while (s.Peek() >= '0' && s.Peek() <= '9') { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (significandDigit < 17) { d = d * 10.0 + (s.TakePush() - '0'); --expFrac; - if (d > 0.0) + if (RAPIDJSON_LIKELY(d > 0.0)) significandDigit++; } else @@ -908,38 +1618,35 @@ private: // Parse exp = e [ minus / plus ] 1*DIGIT int exp = 0; - if (s.Peek() == 'e' || s.Peek() == 'E') { + if (Consume(s, 'e') || Consume(s, 'E')) { if (!useDouble) { - d = use64bit ? i64 : i; + d = static_cast(use64bit ? i64 : i); useDouble = true; } - s.Take(); bool expMinus = false; - if (s.Peek() == '+') - s.Take(); - else if (s.Peek() == '-') { - s.Take(); + if (Consume(s, '+')) + ; + else if (Consume(s, '-')) expMinus = true; - } - if (s.Peek() >= '0' && s.Peek() <= '9') { - exp = s.Take() - '0'; + if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = static_cast(s.Take() - '0'); if (expMinus) { - while (s.Peek() >= '0' && s.Peek() <= '9') { - exp = exp * 10 + (s.Take() - '0'); + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); if (exp >= 214748364) { // Issue #313: prevent overflow exponent - while (s.Peek() >= '0' && s.Peek() <= '9') // Consume the rest of exponent + while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent s.Take(); } } } else { // positive exp int maxExp = 308 - expFrac; - while (s.Peek() >= '0' && s.Peek() <= '9') { - exp = exp * 10 + (s.Take() - '0'); - if (exp > maxExp) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (RAPIDJSON_UNLIKELY(exp > maxExp)) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); } } } @@ -952,34 +1659,63 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; - size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - if (useDouble) { - int p = exp + expFrac; - if (parseFlags & kParseFullPrecisionFlag) - d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); - else - d = internal::StrtodNormalPrecision(d, p); - - cont = handler.Double(minus ? -d : d); - } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(-(int64_t)i64); - else - cont = handler.Uint64(i64); + if (parseFlags & kParseNumbersAsStringsFlag) { + if (parseFlags & kParseInsituFlag) { + s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch* head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + // unable to insert the \0 character here, it will erase the comma after this number + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); } else { - if (minus) - cont = handler.Int(-(int)i); - else - cont = handler.Uint(i); + SizeType numCharsToCopy = static_cast(s.Length()); + StringStream srcStream(s.Pop()); + StackStream dstStream(stack_); + while (numCharsToCopy--) { + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + } + dstStream.Put('\0'); + const typename TargetEncoding::Ch* str = dstStream.Pop(); + const SizeType length = static_cast(dstStream.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); } } - if (!cont) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + else { + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); + } + else if (useNanOrInf) { + cont = handler.Double(d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); + } + } + } + if (RAPIDJSON_UNLIKELY(!cont)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); } // Parse any JSON value @@ -992,7 +1728,10 @@ private: case '"': ParseString(is, handler); break; case '{': ParseObject(is, handler); break; case '[': ParseArray (is, handler); break; - default : ParseNumber(is, handler); + default : + ParseNumber(is, handler); + break; + } } @@ -1000,27 +1739,29 @@ private: // States enum IterativeParsingState { - IterativeParsingStartState = 0, - IterativeParsingFinishState, - IterativeParsingErrorState, + IterativeParsingFinishState = 0, // sink states at top + IterativeParsingErrorState, // sink states at top + IterativeParsingStartState, // Object states IterativeParsingObjectInitialState, IterativeParsingMemberKeyState, - IterativeParsingKeyValueDelimiterState, IterativeParsingMemberValueState, - IterativeParsingMemberDelimiterState, IterativeParsingObjectFinishState, // Array states IterativeParsingArrayInitialState, IterativeParsingElementState, - IterativeParsingElementDelimiterState, IterativeParsingArrayFinishState, // Single value state IterativeParsingValueState, - + + // Delimiter states (at bottom) + IterativeParsingElementDelimiterState, + IterativeParsingMemberDelimiterState, + IterativeParsingKeyValueDelimiterState, + cIterativeParsingStateCount }; @@ -1064,9 +1805,9 @@ private: #undef N #undef N16 //!@endcond - - if (sizeof(Ch) == 1 || unsigned(c) < 256) - return (Token)tokenMap[(unsigned char)c]; + + if (sizeof(Ch) == 1 || static_cast(c) < 256) + return static_cast(tokenMap[static_cast(c)]); else return NumberToken; } @@ -1074,6 +1815,18 @@ private: RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { // current state x one lookahead token -> new state static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, // Start { IterativeParsingArrayInitialState, // Left bracket @@ -1088,18 +1841,6 @@ private: IterativeParsingValueState, // Null IterativeParsingValueState // Number }, - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, // ObjectInitial { IterativeParsingErrorState, // Left bracket @@ -1128,20 +1869,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, // MemberValue { IterativeParsingErrorState, // Left bracket @@ -1156,20 +1883,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, // ObjectFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1204,20 +1917,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // ElementDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, // ArrayFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1229,10 +1928,52 @@ private: IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState - } + }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, }; // End of G - return (IterativeParsingState)G[state][token]; + return static_cast(G[state][token]); } // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). @@ -1309,6 +2050,11 @@ private: case IterativeParsingObjectFinishState: { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); + return IterativeParsingErrorState; + } // Get member count. SizeType c = *stack_.template Pop(1); // If the object is not empty, count the last member. @@ -1334,6 +2080,11 @@ private: case IterativeParsingArrayFinishState: { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); + return IterativeParsingErrorState; + } // Get element count. SizeType c = *stack_.template Pop(1); // If the array is not empty, count the last element. @@ -1385,55 +2136,68 @@ private: // Error flag has been set. return; } - + switch (src) { - case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); - case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; case IterativeParsingObjectInitialState: - case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); - case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - } + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; + case IterativeParsingKeyValueDelimiterState: + case IterativeParsingArrayInitialState: + case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; + default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; + } } + RAPIDJSON_FORCEINLINE bool IsIterativeParsingDelimiterState(IterativeParsingState s) { + return s >= IterativeParsingElementDelimiterState; + } + + RAPIDJSON_FORCEINLINE bool IsIterativeParsingCompleteState(IterativeParsingState s) { + return s <= IterativeParsingErrorState; + } + template ParseResult IterativeParse(InputStream& is, Handler& handler) { parseResult_.Clear(); ClearStackOnExit scope(*this); IterativeParsingState state = IterativeParsingStartState; - - SkipWhitespace(is); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); while (is.Peek() != '\0') { Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); IterativeParsingState d = Transit(state, t, n, is, handler); - + if (d == IterativeParsingErrorState) { HandleError(state, is); break; } - + state = d; - + // Do not further consume streams if a root JSON has been parsed. if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) break; - - SkipWhitespace(is); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } - + // Handle the end of file. if (state != IterativeParsingFinishState) HandleError(state, is); - + return parseResult_; } static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. ParseResult parseResult_; + IterativeParsingState state_; }; // class GenericReader //! Reader with UTF8 encoding and default allocator. @@ -1441,6 +2205,11 @@ typedef GenericReader, UTF8<> > Reader; RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/schema.h b/contrib/rapidjson/include/rapidjson/schema.h new file mode 100644 index 000000000..abcf1a102 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/schema.h @@ -0,0 +1,2016 @@ +// Tencent is pleased to support the open source community by making RapidJSON available-> +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License-> You may obtain a copy of the License at +// +// http://opensource->org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// specific language governing permissions and limitations under the License-> + +#ifndef RAPIDJSON_SCHEMA_H_ +#define RAPIDJSON_SCHEMA_H_ + +#include "document.h" +#include "pointer.h" +#include // abs, floor + +#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 +#endif + +#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX +#include "internal/regex.h" +#elif RAPIDJSON_SCHEMA_USE_STDREGEX +#include +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX +#define RAPIDJSON_SCHEMA_HAS_REGEX 1 +#else +#define RAPIDJSON_SCHEMA_HAS_REGEX 0 +#endif + +#ifndef RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_VERBOSE 0 +#endif + +#if RAPIDJSON_SCHEMA_VERBOSE +#include "stringbuffer.h" +#endif + +RAPIDJSON_DIAG_PUSH + +#if defined(__GNUC__) +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(weak-vtables) +RAPIDJSON_DIAG_OFF(exit-time-destructors) +RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) +RAPIDJSON_DIAG_OFF(variadic-macros) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Verbose Utilities + +#if RAPIDJSON_SCHEMA_VERBOSE + +namespace internal { + +inline void PrintInvalidKeyword(const char* keyword) { + printf("Fail keyword: %s\n", keyword); +} + +inline void PrintInvalidKeyword(const wchar_t* keyword) { + wprintf(L"Fail keyword: %ls\n", keyword); +} + +inline void PrintInvalidDocument(const char* document) { + printf("Fail document: %s\n\n", document); +} + +inline void PrintInvalidDocument(const wchar_t* document) { + wprintf(L"Fail document: %ls\n\n", document); +} + +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { + printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +} + +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { + wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +} // namespace internal + +#endif // RAPIDJSON_SCHEMA_VERBOSE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_INVALID_KEYWORD_RETURN + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) +#else +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) +#endif + +#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + context.invalidKeyword = keyword.GetString();\ + RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ + return false;\ +RAPIDJSON_MULTILINEMACRO_END + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations + +template +class GenericSchemaDocument; + +namespace internal { + +template +class Schema; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaValidator + +class ISchemaValidator { +public: + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaStateFactory + +template +class ISchemaStateFactory { +public: + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Hasher + +// For comparison of compound value +template +class Hasher { +public: + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) { + Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) { + WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) { + uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) { + uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const { + RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } + +private: + static const size_t kDefaultSize = 256; + struct Number { + union U { + uint64_t u; + int64_t i; + }u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) { + // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) { + static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidationContext + +template +struct SchemaValidationContext { + typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType { + kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : + factory(f), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() { + if (hasher) + factory.DestroryHasher(hasher); + if (validators) { + for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) { + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Schema + +template +class Schema { +public: + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + typeless_(schemaDocument->GetTypeless()), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false) + { + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) { + type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) { + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; + char buffer[256 + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); + } + } + + if (schemaDocument) { + AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } + + if (const ValueType* v = GetMember(value, GetNotString())) { + schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } + + // Object + + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { + // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); + + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); + + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); + } + + if (allProperties.Size() > 0) { + propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) { + new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = typeless_; + } + } + } + + if (properties && properties->IsObject()) { + PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { + SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { + PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { + new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) { + SizeType index; + if (FindPropertyIndex(*itr, &index)) { + properties_[index].required = true; + hasRequired_ = true; + } + } + + if (dependencies && dependencies->IsObject()) { + PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) { + if (itr->value.IsArray()) { + properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { + SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; + } + } + else if (itr->value.IsObject()) { + hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } + } + } + } + + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { + if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) { + PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) { // Tuple validation + itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } + } + + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); + + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { + if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); + } + + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); + + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); + + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); + + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); + + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); + + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); + + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + } + + ~Schema() { + AllocatorType::Free(enum_); + if (properties_) { + for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); + } + if (patternProperties_) { + for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); + } + AllocatorType::Free(itemsTuple_); +#if RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) { + pattern_->~RegexType(); + AllocatorType::Free(pattern_); + } +#endif + } + + bool BeginValue(Context& context) const { + if (context.inArray) { + if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) { + if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = typeless_; + else + RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + else + context.valueSchema = typeless_; + + context.arrayElementIndex++; + } + return true; + } + + RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + if (context.patternPropertiesValidatorCount > 0) { + bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) { + patternValid = false; + break; + } + + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { + if (!patternValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { + if (!patternValid || !otherValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + + if (enum_) { + const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); + foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + + if (anyOf_.schemas) { + for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); + foundAny:; + } + + if (oneOf_.schemas) { + bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) { + if (oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + else + oneValid = true; + } + if (!oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } + + if (not_ && context.validators[notValidatorIndex_]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + + return true; + } + + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Int(Context& context, int i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint(Context& context, unsigned u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint64(Context& context, uint64_t u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Double(Context& context, double d) const { + if (!(type_ & (1 << kNumberSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; + + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; + + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; + + return CreateParallelValidator(context); + } + + bool String(Context& context, const Ch* str, SizeType length, bool) const { + if (!(type_ & (1 << kStringSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) { + SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) { + if (count < minLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + if (count > maxLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (hasDependencies_ || hasRequired_) { + context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) { // pre-allocate schema array + SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const { + if (patternProperties_) { + context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + context.valueSchema = typeless_; + } + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { + if (context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) { + if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) { + context.valueSchema = typeless_; + return true; + } + + if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + + return true; + } + + bool EndObject(Context& context, SizeType memberCount) const { + if (hasRequired_) + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required) + if (!context.propertyExist[index]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + + if (memberCount < minProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + + if (memberCount > maxProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + + if (hasDependencies_) { + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) + if (context.propertyExist[sourceIndex]) { + if (properties_[sourceIndex].dependencies) { + for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + else if (properties_[sourceIndex].dependenciesSchema) + if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + } + + return true; + } + + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + context.arrayElementIndex = 0; + context.inArray = true; + + return CreateParallelValidator(context); + } + + bool EndArray(Context& context, SizeType elementCount) const { + context.inArray = false; + + if (elementCount < minItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + + if (elementCount > maxItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + + return true; + } + + // Generate functions for string literal according to Ch +#define RAPIDJSON_STRING_(name, ...) \ + static const ValueType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ + return v;\ + } + + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + RAPIDJSON_STRING_(Not, 'n', 'o', 't') + RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + +#undef RAPIDJSON_STRING_ + +private: + enum SchemaValueType { + kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + typedef internal::GenericRegex RegexType; +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + typedef std::basic_regex RegexType; +#else + typedef char RegexType; +#endif + + struct SchemaArray { + SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + template + void AddUniqueElement(V1& a, const V2& v) { + for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) { + typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { + if (const ValueType* v = GetMember(value, name)) { + if (v->IsArray() && v->Size() > 0) { + PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); + if (!r->IsValid()) { + r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { + GenericRegexSearch rs(*pattern); + return rs.Search(str); + } +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) + try { + return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) { + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { + std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } +#else + template + RegexType* CreatePattern(const ValueType&) { return 0; } + + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } +#endif // RAPIDJSON_SCHEMA_USE_STDREGEX + + void AddType(const ValueType& type) { + if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const { + if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) { + RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) { + for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { + for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { + SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { + *outIndex = index; + return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsInt64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsUint64()) { + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsInt64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsUint64()) { } + /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckUint(Context& context, uint64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsUint64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsUint64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (i % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const { + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const { + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const { + double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + return true; + } + + struct Property { + Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty { + PatternProperty() : schema(), pattern() {} + ~PatternProperty() { + if (pattern) { + pattern->~RegexType(); + AllocatorType::Free(pattern); + } + } + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + const SchemaType* typeless_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; +}; + +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = static_cast(buffer[i]); + } +}; + +// Partial specialized version for char to prevent buffer copying. +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + if (sizeof(SizeType) == 4) { + char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else { + char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// IGenericRemoteSchemaDocumentProvider + +template +class IGenericRemoteSchemaDocumentProvider { +public: + typedef typename SchemaDocumentType::Ch Ch; + + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaDocument + +//! JSON schema document. +/*! + A JSON schema document is a compiled version of a JSON schema. + It is basically a tree of internal::Schema. + + \note This is an immutable class (i.e. its instance cannot be modified after construction). + \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. + \tparam Allocator Allocator type for allocating memory of this document. +*/ +template +class GenericSchemaDocument { +public: + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + typeless_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + + typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); + new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) { + SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) { + if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) { + new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); + } + } + else if (refEntry->schema) + *refEntry->schema = typeless_; + + refEntry->~SchemaRefEntry(); + } + + RAPIDJSON_ASSERT(root_ != 0); + + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + typeless_(rhs.typeless_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)) + { + rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.typeless_ = 0; + } +#endif + + //! Destructor + ~GenericSchemaDocument() { + while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); + + if (typeless_) { + typeless_->~SchemaType(); + Allocator::Free(typeless_); + } + + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } + +private: + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry { + SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry { + SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() { + if (owned) { + schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + if (schema) + *schema = typeless_; + + if (v.GetType() == kObjectType) { + const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) { + if (!HandleRefSchema(pointer, schema, v, document)) { + SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); + if (schema) + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { + static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) { + SizeType len = itr->value.GetStringLength(); + if (len > 0) { + const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) { // Remote reference, resolve immediately + if (remoteProvider_) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { + if (schema) + *schema = sc; + return true; + } + } + } + } + } + else if (s[i] == '#') { // Local reference, defer resolution + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; + + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + const SchemaType* GetTypeless() const { return typeless_; } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + SchemaType* typeless_; + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref +}; + +//! GenericSchemaDocument using Value type. +typedef GenericSchemaDocument SchemaDocument; +//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaValidator + +//! JSON Schema Validator. +/*! + A SAX style JSON schema validator. + It uses a \c GenericSchemaDocument to validate SAX events. + It delegates the incoming SAX events to an output handler. + The default output handler does nothing. + It can be reused multiple times by calling \c Reset(). + + \tparam SchemaDocumentType Type of schema document. + \tparam OutputHandler Type of output handler. Default handler does nothing. + \tparam StateAllocator Allocator for storing the internal validation states. +*/ +template < + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> +class GenericSchemaValidator : + public internal::ISchemaStateFactory, + public internal::ISchemaValidator +{ +public: + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(&outputHandler), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Destructor. + ~GenericSchemaValidator() { + Reset(); + RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() { + while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const { + return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const { + return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const { + return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + internal::PrintInvalidDocument(documentStack_.template Bottom());\ +RAPIDJSON_MULTILINEMACRO_END +#else +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() +#endif + +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ + if (!valid_) return false; \ + if (!BeginValue() || !CurrentSchema().method arg1) {\ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ + return valid_ = false;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ + for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ + if (context->hasher)\ + static_cast(context->hasher)->method arg2;\ + if (context->validators)\ + for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ + static_cast(context->validators[i_])->method arg2;\ + if (context->patternPropertiesValidators)\ + for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ + static_cast(context->patternPropertiesValidators[i_])->method arg2;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ + return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) + +#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ + RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) + + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } + bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = !outputHandler_ || outputHandler_->StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) { + if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = !outputHandler_ || outputHandler_->StartArray(); + } + + bool EndArray(SizeType elementCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } + +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ +#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ +#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ + + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, +#if RAPIDJSON_SCHEMA_VERBOSE + depth_ + 1, +#endif + &GetStateAllocator()); + } + + virtual void DestroySchemaValidator(ISchemaValidator* validator) { + GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } + + virtual void* CreateHasher() { + return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } + + virtual uint64_t GetHashCode(void* hasher) { + return static_cast(hasher)->GetHashCode(); + } + + virtual void DestroryHasher(void* hasher) { + HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } + + virtual void* MallocState(size_t size) { + return GetStateAllocator().Malloc(size); + } + + virtual void FreeState(void* p) { + StateAllocator::Free(p); + } + +private: + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; + + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth, +#endif + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(depth) +#endif + { + } + + StateAllocator& GetStateAllocator() { + if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator)(); + return *stateAllocator_; + } + + bool BeginValue() { + if (schemaStack_.Empty()) + PushSchema(root_); + else { + if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; + + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + RAPIDJSON_ASSERT(CurrentContext().valueSchema); + PushSchema(*CurrentContext().valueSchema); + + if (count > 0) { + CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } + + CurrentContext().arrayUniqueness = valueUniqueness; + } + return true; + } + + bool EndValue() { + if (!CurrentSchema().EndValue(CurrentContext())) + return false; + +#if RAPIDJSON_SCHEMA_VERBOSE + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); +#endif + + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) { + Context& context = CurrentContext(); + if (context.valueUniqueness) { + HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) + RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) { + documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) { + if (str[i] == '~') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } + + RAPIDJSON_FORCEINLINE void PopSchema() { + Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { + a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + OutputHandler* outputHandler_; + bool valid_; +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth_; +#endif +}; + +typedef GenericSchemaValidator SchemaValidator; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidatingReader + +//! A helper class for parsing with validation. +/*! + This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). + + \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam SourceEncoding Encoding of the input stream. + \tparam SchemaDocumentType Type of schema document. + \tparam StackAllocator Allocator type for stack. +*/ +template < + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader { +public: + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} + + template + bool operator()(Handler& handler) { + GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) { + invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + } + else { + invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + +private: + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + bool isValid_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_SCHEMA_H_ diff --git a/contrib/rapidjson/include/rapidjson/stream.h b/contrib/rapidjson/include/rapidjson/stream.h new file mode 100644 index 000000000..fef82c252 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/stream.h @@ -0,0 +1,179 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "rapidjson.h" + +#ifndef RAPIDJSON_STREAM_H_ +#define RAPIDJSON_STREAM_H_ + +#include "encodings.h" + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Reserve n characters for writing to a stream. +template +inline void PutReserve(Stream& stream, size_t count) { + (void)stream; + (void)count; +} + +//! Write character to a stream, presuming buffer is reserved. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { + stream.Put(c); +} + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + PutReserve(stream, n); + for (size_t i = 0; i < n; i++) + PutUnsafe(stream, c); +} + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_STREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/stringbuffer.h b/contrib/rapidjson/include/rapidjson/stringbuffer.h index 1c9c80b79..4e38b82c3 100644 --- a/contrib/rapidjson/include/rapidjson/stringbuffer.h +++ b/contrib/rapidjson/include/rapidjson/stringbuffer.h @@ -15,7 +15,8 @@ #ifndef RAPIDJSON_STRINGBUFFER_H_ #define RAPIDJSON_STRINGBUFFER_H_ -#include "rapidjson.h" +#include "stream.h" +#include "internal/stack.h" #if RAPIDJSON_HAS_CXX11_RVALUE_REFS #include // std::move @@ -23,6 +24,11 @@ #include "internal/stack.h" +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Represents an in-memory output stream. @@ -48,6 +54,7 @@ public: #endif void Put(Ch c) { *stack_.template Push() = c; } + void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } void Flush() {} void Clear() { stack_.Clear(); } @@ -57,7 +64,10 @@ public: stack_.ShrinkToFit(); stack_.template Pop(1); } + + void Reserve(size_t count) { stack_.template Reserve(count); } Ch* Push(size_t count) { return stack_.template Push(count); } + Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } void Pop(size_t count) { stack_.template Pop(count); } const Ch* GetString() const { @@ -68,8 +78,12 @@ public: return stack_.template Bottom(); } + //! Get the size of string in bytes in the string buffer. size_t GetSize() const { return stack_.GetSize(); } + //! Get the length of string in Ch in the string buffer. + size_t GetLength() const { return stack_.GetSize() / sizeof(Ch); } + static const size_t kDefaultCapacity = 256; mutable internal::Stack stack_; @@ -82,6 +96,16 @@ private: //! String buffer with UTF8 encoding typedef GenericStringBuffer > StringBuffer; +template +inline void PutReserve(GenericStringBuffer& stream, size_t count) { + stream.Reserve(count); +} + +template +inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { + stream.PutUnsafe(c); +} + //! Implement specialized version of PutN() with memset() for better performance. template<> inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { @@ -90,4 +114,8 @@ inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/contrib/rapidjson/include/rapidjson/writer.h b/contrib/rapidjson/include/rapidjson/writer.h index e1eea38b9..e610ebb60 100644 --- a/contrib/rapidjson/include/rapidjson/writer.h +++ b/contrib/rapidjson/include/rapidjson/writer.h @@ -15,7 +15,8 @@ #ifndef RAPIDJSON_WRITER_H_ #define RAPIDJSON_WRITER_H_ -#include "rapidjson.h" +#include "stream.h" +#include "internal/meta.h" #include "internal/stack.h" #include "internal/strfunc.h" #include "internal/dtoa.h" @@ -23,8 +24,16 @@ #include "stringbuffer.h" #include // placement new -#if RAPIDJSON_HAS_STDSTRING -#include +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#elif defined(RAPIDJSON_NEON) +#include #endif #ifdef _MSC_VER @@ -32,8 +41,36 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN +/////////////////////////////////////////////////////////////////////////////// +// WriteFlag + +/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kWriteDefaultFlags definition. + + User can define this as any \c WriteFlag combinations. +*/ +#ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS +#define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags +#endif + +//! Combination of writeFlags +enum WriteFlag { + kWriteNoFlags = 0, //!< No flags are set. + kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. + kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. + kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS +}; + //! JSON writer /*! Writer implements the concept Handler. It generates JSON text by events to an output os. @@ -50,11 +87,13 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam StackAllocator Type of allocator for allocating memory of stack. \note implements Handler concept */ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> class Writer { public: typedef typename SourceEncoding::Ch Ch; + static const int kDefaultMaxDecimalPlaces = 324; + //! Constructor /*! \param os Output stream. \param stackAllocator User supplied allocator. If it is null, it will create a private one. @@ -62,11 +101,18 @@ public: */ explicit Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), hasRoot_(false) {} + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} explicit Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), hasRoot_(false) {} + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Writer(Writer&& rhs) : + os_(rhs.os_), level_stack_(std::move(rhs.level_stack_)), maxDecimalPlaces_(rhs.maxDecimalPlaces_), hasRoot_(rhs.hasRoot_) { + rhs.os_ = 0; + } +#endif //! Reset the writer with a new stream. /*! @@ -100,29 +146,66 @@ public: return hasRoot_ && level_stack_.Empty(); } + int GetMaxDecimalPlaces() const { + return maxDecimalPlaces_; + } + + //! Sets the maximum number of decimal places for double output. + /*! + This setting truncates the output with specified number of decimal places. + + For example, + + \code + writer.SetMaxDecimalPlaces(3); + writer.StartArray(); + writer.Double(0.12345); // "0.123" + writer.Double(0.0001); // "0.0" + writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) + writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) + writer.EndArray(); + \endcode + + The default setting does not truncate any decimal places. You can restore to this setting by calling + \code + writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); + \endcode + */ + void SetMaxDecimalPlaces(int maxDecimalPlaces) { + maxDecimalPlaces_ = maxDecimalPlaces; + } + /*!@name Implementation of Handler \see Handler */ //@{ - bool Null() { Prefix(kNullType); return WriteNull(); } - bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } - bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } - bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } - bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } - bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } + bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } + bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } + bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } + bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } //! Writes the given \c double value to the stream /*! \param d The value to be written. \return Whether it is succeed. */ - bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } + bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); + (void)copy; + Prefix(kNumberType); + return EndValue(WriteString(str, length)); + } bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; Prefix(kStringType); - return WriteString(str, length); + return EndValue(WriteString(str, length)); } #if RAPIDJSON_HAS_STDSTRING @@ -138,16 +221,21 @@ public: } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) + { + return Key(str.data(), SizeType(str.size())); + } +#endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value level_stack_.template Pop(1); - bool ret = WriteEndObject(); - if (level_stack_.Empty()) // end of json text - os_->Flush(); - return ret; + return EndValue(WriteEndObject()); } bool StartArray() { @@ -161,10 +249,7 @@ public: RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); level_stack_.template Pop(1); - bool ret = WriteEndArray(); - if (level_stack_.Empty()) // end of json text - os_->Flush(); - return ret; + return EndValue(WriteEndArray()); } //@} @@ -172,11 +257,33 @@ public: //@{ //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - + bool String(const Ch* const& str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* const& str) { return Key(str, internal::StrLen(str)); } + //@} + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + Prefix(type); + return EndValue(WriteRawValue(json, length)); + } + + //! Flush the output stream. + /*! + Allows the user to flush the output stream immediately. + */ + void Flush() { + os_->Flush(); + } + protected: //! Information for each nested level struct Level { @@ -188,15 +295,18 @@ protected: static const size_t kDefaultLevelDepth = 32; bool WriteNull() { - os_->Put('n'); os_->Put('u'); os_->Put('l'); os_->Put('l'); return true; + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; } bool WriteBool(bool b) { if (b) { - os_->Put('t'); os_->Put('r'); os_->Put('u'); os_->Put('e'); + PutReserve(*os_, 4); + PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); } else { - os_->Put('f'); os_->Put('a'); os_->Put('l'); os_->Put('s'); os_->Put('e'); + PutReserve(*os_, 5); + PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); } return true; } @@ -204,45 +314,69 @@ protected: bool WriteInt(int i) { char buffer[11]; const char* end = internal::i32toa(i, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteUint(unsigned u) { char buffer[10]; const char* end = internal::u32toa(u, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteInt64(int64_t i64) { char buffer[21]; const char* end = internal::i64toa(i64, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteUint64(uint64_t u64) { char buffer[20]; char* end = internal::u64toa(u64, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + if (!(writeFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + char buffer[25]; - char* end = internal::dtoa(d, buffer); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteString(const Ch* str, SizeType length) { - static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const typename OutputStream::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; static const char escape[256] = { #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 //0 1 2 3 4 5 6 7 8 9 A B C D E F @@ -255,22 +389,27 @@ protected: #undef Z16 }; - os_->Put('\"'); + if (TargetEncoding::supportUnicode) + PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + else + PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." + + PutUnsafe(*os_, '\"'); GenericStringStream is(str); - while (is.Tell() < length) { + while (ScanWriteUnescapedString(is, length)) { const Ch c = is.Peek(); - if (!TargetEncoding::supportUnicode && (unsigned)c >= 0x80) { + if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { // Unicode escaping unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) + if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) return false; - os_->Put('\\'); - os_->Put('u'); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { - os_->Put(hexDigits[(codepoint >> 12) & 15]); - os_->Put(hexDigits[(codepoint >> 8) & 15]); - os_->Put(hexDigits[(codepoint >> 4) & 15]); - os_->Put(hexDigits[(codepoint ) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); } else { RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); @@ -278,45 +417,59 @@ protected: unsigned s = codepoint - 0x010000; unsigned lead = (s >> 10) + 0xD800; unsigned trail = (s & 0x3FF) + 0xDC00; - os_->Put(hexDigits[(lead >> 12) & 15]); - os_->Put(hexDigits[(lead >> 8) & 15]); - os_->Put(hexDigits[(lead >> 4) & 15]); - os_->Put(hexDigits[(lead ) & 15]); - os_->Put('\\'); - os_->Put('u'); - os_->Put(hexDigits[(trail >> 12) & 15]); - os_->Put(hexDigits[(trail >> 8) & 15]); - os_->Put(hexDigits[(trail >> 4) & 15]); - os_->Put(hexDigits[(trail ) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(lead ) & 15]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(trail ) & 15]); } } - else if ((sizeof(Ch) == 1 || (unsigned)c < 256) && escape[(unsigned char)c]) { + else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { is.Take(); - os_->Put('\\'); - os_->Put(escape[(unsigned char)c]); - if (escape[(unsigned char)c] == 'u') { - os_->Put('0'); - os_->Put('0'); - os_->Put(hexDigits[(unsigned char)c >> 4]); - os_->Put(hexDigits[(unsigned char)c & 0xF]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + if (escape[static_cast(c)] == 'u') { + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); + PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); } } - else - if (!Transcoder::Transcode(is, *os_)) - return false; + else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; } - os_->Put('\"'); + PutUnsafe(*os_, '\"'); return true; } + bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { + return RAPIDJSON_LIKELY(is.Tell() < length); + } + bool WriteStartObject() { os_->Put('{'); return true; } bool WriteEndObject() { os_->Put('}'); return true; } bool WriteStartArray() { os_->Put('['); return true; } bool WriteEndArray() { os_->Put(']'); return true; } + bool WriteRawValue(const Ch* json, size_t length) { + PutReserve(*os_, length); + for (size_t i = 0; i < length; i++) { + RAPIDJSON_ASSERT(json[i] != '\0'); + PutUnsafe(*os_, json[i]); + } + return true; + } + void Prefix(Type type) { (void)type; - if (level_stack_.GetSize() != 0) { // this value is not at root + if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root Level* level = level_stack_.template Top(); if (level->valueCount > 0) { if (level->inArray) @@ -334,8 +487,16 @@ protected: } } + // Flush the value if it is the top level one. + bool EndValue(bool ret) { + if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + Flush(); + return ret; + } + OutputStream* os_; internal::Stack level_stack_; + int maxDecimalPlaces_; bool hasRoot_; private: @@ -350,7 +511,7 @@ template<> inline bool Writer::WriteInt(int i) { char *buffer = os_->Push(11); const char* end = internal::i32toa(i, buffer); - os_->Pop(11 - (end - buffer)); + os_->Pop(static_cast(11 - (end - buffer))); return true; } @@ -358,7 +519,7 @@ template<> inline bool Writer::WriteUint(unsigned u) { char *buffer = os_->Push(10); const char* end = internal::u32toa(u, buffer); - os_->Pop(10 - (end - buffer)); + os_->Pop(static_cast(10 - (end - buffer))); return true; } @@ -366,7 +527,7 @@ template<> inline bool Writer::WriteInt64(int64_t i64) { char *buffer = os_->Push(21); const char* end = internal::i64toa(i64, buffer); - os_->Pop(21 - (end - buffer)); + os_->Pop(static_cast(21 - (end - buffer))); return true; } @@ -374,22 +535,177 @@ template<> inline bool Writer::WriteUint64(uint64_t u) { char *buffer = os_->Push(20); const char* end = internal::u64toa(u, buffer); - os_->Pop(20 - (end - buffer)); + os_->Pop(static_cast(20 - (end - buffer))); return true; } template<> inline bool Writer::WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). + if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + char *buffer = os_->Push(25); - char* end = internal::dtoa(d, buffer); - os_->Pop(25 - (end - buffer)); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + os_->Pop(static_cast(25 - (end - buffer))); return true; } +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (; p != endAligned; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType len; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + len = offset; +#else + len = static_cast(__builtin_ffs(r) - 1); +#endif + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#elif defined(RAPIDJSON_NEON) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (; p != endAligned; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType len = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + len = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + len = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + vst1q_u8(reinterpret_cast(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#endif // RAPIDJSON_NEON + RAPIDJSON_NAMESPACE_END #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/contrib/rapidjson/license.txt b/contrib/rapidjson/license.txt index 879293afa..7ccc161c8 100644 --- a/contrib/rapidjson/license.txt +++ b/contrib/rapidjson/license.txt @@ -3,7 +3,7 @@ Tencent is pleased to support the open source community by making RapidJSON avai Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. +If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. A copy of the MIT License is included in this file. Other dependencies and licenses: diff --git a/contrib/rapidjson/readme.md b/contrib/rapidjson/readme.md index 19da38667..b833a98e8 100644 --- a/contrib/rapidjson/readme.md +++ b/contrib/rapidjson/readme.md @@ -1,17 +1,17 @@ -![](doc/logo/rapidjson.png) +![RapidJSON logo](doc/logo/rapidjson.png) -![](https://img.shields.io/badge/release-v1.0.2-blue.png) +![Release version](https://img.shields.io/badge/release-v1.1.0-blue.svg) -## A fast JSON parser/generator for C++ with both SAX/DOM style API +## A fast JSON parser/generator for C++ with both SAX/DOM style API Tencent is pleased to support the open source community by making RapidJSON available. Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -* [RapidJSON GitHub](https://github.com/miloyip/rapidjson/) +* [RapidJSON GitHub](https://github.com/Tencent/rapidjson/) * RapidJSON Documentation - * [English](http://miloyip.github.io/rapidjson/) - * [简体中文](http://miloyip.github.io/rapidjson/zh-cn/) + * [English](http://rapidjson.org/) + * [简体中文](http://rapidjson.org/zh-cn/) * [GitBook](https://www.gitbook.com/book/miloyip/rapidjson/) with downloadable PDF/EPUB/MOBI, without API reference. ## Build status @@ -20,33 +20,43 @@ Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights | :---------------: | :-----------------: | :-------------------: | | ![lin-badge] | ![win-badge] | ![cov-badge] | -[lin-badge]: https://travis-ci.org/miloyip/rapidjson.png?branch=master "Travis build status" -[lin-link]: https://travis-ci.org/miloyip/rapidjson "Travis build status" -[win-badge]: https://ci.appveyor.com/api/projects/status/u658dcuwxo14a8m9/branch/master "AppVeyor build status" -[win-link]: https://ci.appveyor.com/project/miloyip/rapidjson/branch/master "AppVeyor build status" -[cov-badge]: https://coveralls.io/repos/miloyip/rapidjson/badge.png?branch=master -[cov-link]: https://coveralls.io/r/miloyip/rapidjson?branch=master +[lin-badge]: https://travis-ci.org/Tencent/rapidjson.svg?branch=master "Travis build status" +[lin-link]: https://travis-ci.org/Tencent/rapidjson "Travis build status" +[win-badge]: https://ci.appveyor.com/api/projects/status/l6qulgqahcayidrf/branch/master?svg=true "AppVeyor build status" +[win-link]: https://ci.appveyor.com/project/miloyip/rapidjson-0fdqj/branch/master "AppVeyor build status" +[cov-badge]: https://coveralls.io/repos/Tencent/rapidjson/badge.svg?branch=master "Coveralls coverage" +[cov-link]: https://coveralls.io/r/Tencent/rapidjson?branch=master "Coveralls coverage" ## Introduction RapidJSON is a JSON parser and generator for C++. It was inspired by [RapidXml](http://rapidxml.sourceforge.net/). -* RapidJSON is small but complete. It supports both SAX and DOM style API. The SAX parser is only a half thousand lines of code. +* RapidJSON is **small** but **complete**. It supports both SAX and DOM style API. The SAX parser is only a half thousand lines of code. -* RapidJSON is fast. Its performance can be comparable to `strlen()`. It also optionally supports SSE2/SSE4.2 for acceleration. +* RapidJSON is **fast**. Its performance can be comparable to `strlen()`. It also optionally supports SSE2/SSE4.2 for acceleration. -* RapidJSON is self-contained. It does not depend on external libraries such as BOOST. It even does not depend on STL. +* RapidJSON is **self-contained** and **header-only**. It does not depend on external libraries such as BOOST. It even does not depend on STL. -* RapidJSON is memory friendly. Each JSON value occupies exactly 16/20 bytes for most 32/64-bit machines (excluding text string). By default it uses a fast memory allocator, and the parser allocates memory compactly during parsing. +* RapidJSON is **memory-friendly**. Each JSON value occupies exactly 16 bytes for most 32/64-bit machines (excluding text string). By default it uses a fast memory allocator, and the parser allocates memory compactly during parsing. -* RapidJSON is Unicode friendly. It supports UTF-8, UTF-16, UTF-32 (LE & BE), and their detection, validation and transcoding internally. For example, you can read a UTF-8 file and let RapidJSON transcode the JSON strings into UTF-16 in the DOM. It also supports surrogates and "\u0000" (null character). +* RapidJSON is **Unicode-friendly**. It supports UTF-8, UTF-16, UTF-32 (LE & BE), and their detection, validation and transcoding internally. For example, you can read a UTF-8 file and let RapidJSON transcode the JSON strings into UTF-16 in the DOM. It also supports surrogates and "\u0000" (null character). More features can be read [here](doc/features.md). -JSON(JavaScript Object Notation) is a light-weight data exchange format. RapidJSON should be in fully compliance with RFC7159/ECMA-404. More information about JSON can be obtained at +JSON(JavaScript Object Notation) is a light-weight data exchange format. RapidJSON should be in fully compliance with RFC7159/ECMA-404, with optional support of relaxed syntax. More information about JSON can be obtained at * [Introducing JSON](http://json.org/) -* [RFC7159: The JavaScript Object Notation (JSON) Data Interchange Format](http://www.ietf.org/rfc/rfc7159.txt) -* [Standard ECMA-404: The JSON Data Interchange Format](http://www.ecma-international.org/publications/standards/Ecma-404.htm) +* [RFC7159: The JavaScript Object Notation (JSON) Data Interchange Format](https://tools.ietf.org/html/rfc7159) +* [Standard ECMA-404: The JSON Data Interchange Format](https://www.ecma-international.org/publications/standards/Ecma-404.htm) + +## Highlights in v1.1 (2016-8-25) + +* Added [JSON Pointer](doc/pointer.md) +* Added [JSON Schema](doc/schema.md) +* Added [relaxed JSON syntax](doc/dom.md) (comment, trailing comma, NaN/Infinity) +* Iterating array/object with [C++11 Range-based for loop](doc/tutorial.md) +* Reduce memory overhead of each `Value` from 24 bytes to 16 bytes in x86-64 architecture. + +For other changes please refer to [change log](CHANGELOG.md). ## Compatibility @@ -63,9 +73,9 @@ Users can build and run the unit tests on their platform/compiler. RapidJSON is a header-only C++ library. Just copy the `include/rapidjson` folder to system or project's include path. RapidJSON uses following software as its dependencies: -* [CMake](http://www.cmake.org) as a general build tool -* (optional)[Doxygen](http://www.doxygen.org) to build documentation -* (optional)[googletest](https://code.google.com/p/googletest/) for unit and performance testing +* [CMake](https://cmake.org/) as a general build tool +* (optional) [Doxygen](http://www.doxygen.org) to build documentation +* (optional) [googletest](https://github.com/google/googletest) for unit and performance testing To generate user documentation and run tests please proceed with the steps below: @@ -74,7 +84,7 @@ To generate user documentation and run tests please proceed with the steps below 3. Change to `build` directory and run `cmake ..` command to configure your build. Windows users can do the same with cmake-gui application. 4. On Windows, build the solution found in the build directory. On Linux, run `make` from the build directory. -On successfull build you will find compiled test and example binaries in `bin` +On successful build you will find compiled test and example binaries in `bin` directory. The generated documentation will be available in `doc/html` directory of the build tree. To run tests after finished build please run `make test` or `ctest` from your build tree. You can get detailed output using `ctest @@ -126,4 +136,25 @@ The following diagram shows the process. ![simpledom](doc/diagram/simpledom.png) -More [examples](https://github.com/miloyip/rapidjson/tree/master/example) are available. +More [examples](https://github.com/Tencent/rapidjson/tree/master/example) are available: + +* DOM API + * [tutorial](https://github.com/Tencent/rapidjson/blob/master/example/tutorial/tutorial.cpp): Basic usage of DOM API. + +* SAX API + * [simplereader](https://github.com/Tencent/rapidjson/blob/master/example/simplereader/simplereader.cpp): Dumps all SAX events while parsing a JSON by `Reader`. + * [condense](https://github.com/Tencent/rapidjson/blob/master/example/condense/condense.cpp): A command line tool to rewrite a JSON, with all whitespaces removed. + * [pretty](https://github.com/Tencent/rapidjson/blob/master/example/pretty/pretty.cpp): A command line tool to rewrite a JSON with indents and newlines by `PrettyWriter`. + * [capitalize](https://github.com/Tencent/rapidjson/blob/master/example/capitalize/capitalize.cpp): A command line tool to capitalize strings in JSON. + * [messagereader](https://github.com/Tencent/rapidjson/blob/master/example/messagereader/messagereader.cpp): Parse a JSON message with SAX API. + * [serialize](https://github.com/Tencent/rapidjson/blob/master/example/serialize/serialize.cpp): Serialize a C++ object into JSON with SAX API. + * [jsonx](https://github.com/Tencent/rapidjson/blob/master/example/jsonx/jsonx.cpp): Implements a `JsonxWriter` which stringify SAX events into [JSONx](https://www-01.ibm.com/support/knowledgecenter/SS9H2Y_7.1.0/com.ibm.dp.doc/json_jsonx.html) (a kind of XML) format. The example is a command line tool which converts input JSON into JSONx format. + +* Schema + * [schemavalidator](https://github.com/Tencent/rapidjson/blob/master/example/schemavalidator/schemavalidator.cpp) : A command line tool to validate a JSON with a JSON schema. + +* Advanced + * [prettyauto](https://github.com/Tencent/rapidjson/blob/master/example/prettyauto/prettyauto.cpp): A modified version of [pretty](https://github.com/Tencent/rapidjson/blob/master/example/pretty/pretty.cpp) to automatically handle JSON with any UTF encodings. + * [parsebyparts](https://github.com/Tencent/rapidjson/blob/master/example/parsebyparts/parsebyparts.cpp): Implements an `AsyncDocumentParser` which can parse JSON in parts, using C++11 thread. + * [filterkey](https://github.com/Tencent/rapidjson/blob/master/example/filterkey/filterkey.cpp): A command line tool to remove all values with user-specified key. + * [filterkeydom](https://github.com/Tencent/rapidjson/blob/master/example/filterkeydom/filterkeydom.cpp): Same tool as above, but it demonstrates how to use a generator to populate a `Document`. From 9bcfce63dcb706b8aba3748fbd4321506a67a0a0 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:33:27 +0300 Subject: [PATCH 228/305] utMetadata: Fix memory leak --- test/unit/utMetadata.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/utMetadata.cpp b/test/unit/utMetadata.cpp index b44de1eb2..4109b068c 100644 --- a/test/unit/utMetadata.cpp +++ b/test/unit/utMetadata.cpp @@ -79,6 +79,7 @@ TEST_F( utMetadata, allocTest ) { EXPECT_EQ( 1U, data->mNumProperties ); EXPECT_NE( nullptr, data->mKeys ); EXPECT_NE( nullptr, data->mValues ); + aiMetadata::Dealloc( data ); } TEST_F( utMetadata, get_set_pod_Test ) { From 674fb5a46cf5b0202d13972acda9ef308f0c91f7 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:55:58 +0300 Subject: [PATCH 229/305] utRemoveVCProcess: Fix memory leak --- test/unit/utRemoveVCProcess.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/utRemoveVCProcess.cpp b/test/unit/utRemoveVCProcess.cpp index c78e80d3f..6caa72c11 100644 --- a/test/unit/utRemoveVCProcess.cpp +++ b/test/unit/utRemoveVCProcess.cpp @@ -70,7 +70,7 @@ TEST_F( utRevmoveVCProcess, issue1266_ProcessMeshTest_NoCrash ) { mesh->mNumVertices = 1; mesh->mColors[ 0 ] = new aiColor4D[ 2 ]; scene->mMeshes[ 0 ] = mesh; - RemoveVCProcess *process = new RemoveVCProcess; + std::unique_ptr process(new RemoveVCProcess); process->Execute( scene ); delete scene; } From 34acf47acda9067397dfa680ace52ae1bb4b7960 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:56:13 +0300 Subject: [PATCH 230/305] UnrealLoader: Fix IOStream leak --- code/UnrealLoader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/UnrealLoader.cpp b/code/UnrealLoader.cpp index c8382cb01..a79a2a5c5 100644 --- a/code/UnrealLoader.cpp +++ b/code/UnrealLoader.cpp @@ -157,7 +157,7 @@ void UnrealImporter::InternReadFile( const std::string& pFile, DefaultLogger::get()->debug("UNREAL: uc file is " + uc_path); // and open the files ... we can't live without them - IOStream* p = pIOHandler->Open(d_path); + std::unique_ptr p(pIOHandler->Open(d_path)); if (!p) throw DeadlyImportError("UNREAL: Unable to open _d file"); StreamReaderLE d_reader(pIOHandler->Open(d_path)); @@ -203,7 +203,7 @@ void UnrealImporter::InternReadFile( const std::string& pFile, d_reader.IncPtr(1); } - p = pIOHandler->Open(a_path); + p.reset(pIOHandler->Open(a_path)); if (!p) throw DeadlyImportError("UNREAL: Unable to open _a file"); StreamReaderLE a_reader(pIOHandler->Open(a_path)); From 316046f74826404783228bbae85edeaff927811d Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:09:26 +0300 Subject: [PATCH 231/305] AMFImporter: Fix memory leak --- code/AMFImporter_Postprocess.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/code/AMFImporter_Postprocess.cpp b/code/AMFImporter_Postprocess.cpp index 085336c51..789a11eb8 100644 --- a/code/AMFImporter_Postprocess.cpp +++ b/code/AMFImporter_Postprocess.cpp @@ -686,7 +686,6 @@ std::list mesh_idx; tmesh->mNumVertices = static_cast(vert_arr.size()); tmesh->mVertices = new aiVector3D[tmesh->mNumVertices]; tmesh->mColors[0] = new aiColor4D[tmesh->mNumVertices]; - tmesh->mFaces = new aiFace[face_list_cur.size()]; memcpy(tmesh->mVertices, vert_arr.data(), tmesh->mNumVertices * sizeof(aiVector3D)); memcpy(tmesh->mColors[0], col_arr.data(), tmesh->mNumVertices * sizeof(aiColor4D)); From b6d2b9179971d8950e4472fc0773f9c30b92da8d Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:35:17 +0300 Subject: [PATCH 232/305] OpenGEXImporter: Store ChildInfo in unique_ptr so they get automatically cleaned up --- code/OpenGEXImporter.cpp | 8 ++++---- code/OpenGEXImporter.h | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index d9c58e91c..0d2ed0520 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -1231,9 +1231,9 @@ void OpenGEXImporter::pushNode( aiNode *node, aiScene *pScene ) { if( m_nodeChildMap.end() == it ) { info = new ChildInfo; m_root = info; - m_nodeChildMap[ node->mParent ] = info; + m_nodeChildMap[ node->mParent ] = std::unique_ptr(info); } else { - info = it->second; + info = it->second.get(); } info->m_children.push_back( node ); } else { @@ -1243,9 +1243,9 @@ void OpenGEXImporter::pushNode( aiNode *node, aiScene *pScene ) { NodeChildMap::iterator it( m_nodeChildMap.find( node->mParent ) ); if( m_nodeChildMap.end() == it ) { info = new ChildInfo; - m_nodeChildMap[ node->mParent ] = info; + m_nodeChildMap[ node->mParent ] = std::unique_ptr(info); } else { - info = it->second; + info = it->second.get(); } info->m_children.push_back( node ); } diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index bb6b45140..948712869 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include namespace ODDLParser { class DDLNode; @@ -179,7 +180,7 @@ private: std::list m_children; }; ChildInfo *m_root; - typedef std::map NodeChildMap; + typedef std::map > NodeChildMap; NodeChildMap m_nodeChildMap; std::vector m_meshCache; From e7ff7b167f270e6eea910ccc18499acfc9a16665 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:38:44 +0300 Subject: [PATCH 233/305] OpenGEXImporter: Fix IOStream leak --- code/OpenGEXImporter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 0d2ed0520..24dabd1d4 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -307,6 +307,7 @@ void OpenGEXImporter::InternReadFile( const std::string &filename, aiScene *pSce std::vector buffer; TextFileToBuffer( file, buffer ); + pIOHandler->Close( file ); OpenDDLParser myParser; myParser.setBuffer( &buffer[ 0 ], buffer.size() ); From 568003a0d3f2f9dce1f7a8173a67d20c216559a4 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:43:49 +0300 Subject: [PATCH 234/305] OpenGEXImporter: Store RefInfo in unique_ptr so they get automatically cleaned up --- code/OpenGEXImporter.cpp | 8 ++++---- code/OpenGEXImporter.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 24dabd1d4..7073a5e76 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -525,7 +525,7 @@ void OpenGEXImporter::handleObjectRefNode( DDLNode *node, aiScene *pScene ) { m_currentNode->mNumMeshes = static_cast(objRefNames.size()); m_currentNode->mMeshes = new unsigned int[ objRefNames.size() ]; if ( !objRefNames.empty() ) { - m_unresolvedRefStack.push_back( new RefInfo( m_currentNode, RefInfo::MeshRef, objRefNames ) ); + m_unresolvedRefStack.push_back( std::unique_ptr( new RefInfo( m_currentNode, RefInfo::MeshRef, objRefNames ) ) ); } } else if ( m_tokenType == Grammar::LightNodeToken ) { // TODO! @@ -544,7 +544,7 @@ void OpenGEXImporter::handleMaterialRefNode( ODDLParser::DDLNode *node, aiScene std::vector matRefNames; getRefNames( node, matRefNames ); if( !matRefNames.empty() ) { - m_unresolvedRefStack.push_back( new RefInfo( m_currentNode, RefInfo::MaterialRef, matRefNames ) ); + m_unresolvedRefStack.push_back( std::unique_ptr( new RefInfo( m_currentNode, RefInfo::MaterialRef, matRefNames ) ) ); } } @@ -1165,8 +1165,8 @@ void OpenGEXImporter::resolveReferences() { } RefInfo *currentRefInfo( nullptr ); - for( std::vector::iterator it = m_unresolvedRefStack.begin(); it != m_unresolvedRefStack.end(); ++it ) { - currentRefInfo = *it; + for( auto it = m_unresolvedRefStack.begin(); it != m_unresolvedRefStack.end(); ++it ) { + currentRefInfo = it->get(); if( nullptr != currentRefInfo ) { aiNode *node( currentRefInfo->m_node ); if( RefInfo::MeshRef == currentRefInfo->m_type ) { diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index 948712869..aefd9ef1d 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -201,7 +201,7 @@ private: std::vector m_cameraCache; std::vector m_lightCache; std::vector m_nodeStack; - std::vector m_unresolvedRefStack; + std::vector > m_unresolvedRefStack; }; } // Namespace OpenGEX From 3f0bb9c63494c097cc3ddf6240af8c76023e7819 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:51:08 +0300 Subject: [PATCH 235/305] OpenGEXImporter: Copy materials to scene --- code/OpenGEXImporter.cpp | 14 ++++++++++++++ code/OpenGEXImporter.h | 1 + 2 files changed, 15 insertions(+) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 7073a5e76..47e3e3e8c 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -322,6 +322,7 @@ void OpenGEXImporter::InternReadFile( const std::string &filename, aiScene *pSce copyMeshes( pScene ); copyCameras( pScene ); copyLights( pScene ); + copyMaterials( pScene ); resolveReferences(); createNodeTree( pScene ); } @@ -1158,6 +1159,19 @@ void OpenGEXImporter::copyLights( aiScene *pScene ) { std::copy( m_lightCache.begin(), m_lightCache.end(), pScene->mLights ); } +//------------------------------------------------------------------------------------------------ +void OpenGEXImporter::copyMaterials( aiScene *pScene ) { + ai_assert( nullptr != pScene ); + + if ( m_materialCache.empty() ) { + return; + } + + pScene->mNumMaterials = static_cast(m_materialCache.size()); + pScene->mMaterials = new aiMaterial*[ pScene->mNumMaterials ]; + std::copy( m_materialCache.begin(), m_materialCache.end(), pScene->mMaterials ); +} + //------------------------------------------------------------------------------------------------ void OpenGEXImporter::resolveReferences() { if( m_unresolvedRefStack.empty() ) { diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index aefd9ef1d..c0cde579c 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -133,6 +133,7 @@ protected: void copyMeshes( aiScene *pScene ); void copyCameras( aiScene *pScene ); void copyLights( aiScene *pScene ); + void copyMaterials( aiScene *pScene ); void resolveReferences(); void pushNode( aiNode *node, aiScene *pScene ); aiNode *popNode(); From 341222697a02f41d6279d256b153a3b341c9e25e Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Thu, 5 Oct 2017 23:55:10 +1100 Subject: [PATCH 236/305] Address warnings on Clang 3.9 caused by having a static data member in a class template. --- code/BlenderBMesh.cpp | 6 +++++- code/BlenderLoader.cpp | 6 +++++- code/BlenderTessellator.cpp | 12 ++++++++++-- code/FBXImporter.cpp | 6 +++++- code/IFCLoader.cpp | 6 +++++- code/LogAux.h | 13 ++++++------- code/XGLLoader.cpp | 7 +++++-- 7 files changed, 41 insertions(+), 15 deletions(-) diff --git a/code/BlenderBMesh.cpp b/code/BlenderBMesh.cpp index c328947ae..8a13819a6 100644 --- a/code/BlenderBMesh.cpp +++ b/code/BlenderBMesh.cpp @@ -52,7 +52,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { - template< > const std::string LogFunctions< BlenderBMeshConverter >::log_prefix = "BLEND_BMESH: "; + template< > const char* LogFunctions< BlenderBMeshConverter >::Prefix() + { + static auto prefix = "BLEND_BMESH: "; + return prefix; + } } using namespace Assimp; diff --git a/code/BlenderLoader.cpp b/code/BlenderLoader.cpp index 1453ef000..6d4c97cbf 100644 --- a/code/BlenderLoader.cpp +++ b/code/BlenderLoader.cpp @@ -74,7 +74,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif namespace Assimp { - template<> const std::string LogFunctions::log_prefix = "BLEND: "; + template<> const char* LogFunctions::Prefix() + { + static auto prefix = "BLEND: "; + return prefix; + } } using namespace Assimp; diff --git a/code/BlenderTessellator.cpp b/code/BlenderTessellator.cpp index 2eaa938dd..9f4c51048 100644 --- a/code/BlenderTessellator.cpp +++ b/code/BlenderTessellator.cpp @@ -59,7 +59,11 @@ static const unsigned int BLEND_TESS_MAGIC = 0x83ed9ac3; namspace Assimp { - template< > const std::string LogFunctions< BlenderTessellatorGL >::log_prefix = "BLEND_TESS_GL: "; + template< > const char* LogFunctions< BlenderTessellatorGL >::Prefix() + { + static auto prefix = "BLEND_TESS_GL: "; + return prefix; + } } using namespace Assimp; @@ -252,7 +256,11 @@ void BlenderTessellatorGL::TessellateError( GLenum errorCode, void* ) namespace Assimp { - template< > const std::string LogFunctions< BlenderTessellatorP2T >::log_prefix = "BLEND_TESS_P2T: "; + template< > const char* LogFunctions< BlenderTessellatorP2T >::Prefix() + { + static auto prefix = "BLEND_TESS_P2T: "; + return prefix; + } } using namespace Assimp; diff --git a/code/FBXImporter.cpp b/code/FBXImporter.cpp index 0ebf13e7d..51e41b8f4 100644 --- a/code/FBXImporter.cpp +++ b/code/FBXImporter.cpp @@ -59,7 +59,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include namespace Assimp { - template<> const std::string LogFunctions::log_prefix = "FBX: "; + template<> const char* LogFunctions::Prefix() + { + static auto prefix = "FBX: "; + return prefix; + } } using namespace Assimp; diff --git a/code/IFCLoader.cpp b/code/IFCLoader.cpp index e8632fa9d..85382d467 100644 --- a/code/IFCLoader.cpp +++ b/code/IFCLoader.cpp @@ -66,7 +66,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { - template<> const std::string LogFunctions::log_prefix = "IFC: "; + template<> const char* LogFunctions::Prefix() + { + static auto prefix = "IFC: "; + return prefix; + } } using namespace Assimp; diff --git a/code/LogAux.h b/code/LogAux.h index 432da5cd5..08b1b3c17 100644 --- a/code/LogAux.h +++ b/code/LogAux.h @@ -60,34 +60,34 @@ public: // ------------------------------------------------------------------------------------------------ static void ThrowException(const std::string& msg) { - throw DeadlyImportError(log_prefix+msg); + throw DeadlyImportError(Prefix()+msg); } // ------------------------------------------------------------------------------------------------ static void LogWarn(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->warn(log_prefix+(std::string)message); + DefaultLogger::get()->warn(Prefix() +(std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogError(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->error(log_prefix+(std::string)message); + DefaultLogger::get()->error(Prefix() +(std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogInfo(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->info(log_prefix+(std::string)message); + DefaultLogger::get()->info(Prefix() +(std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogDebug(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->debug(log_prefix+(std::string)message); + DefaultLogger::get()->debug(Prefix() +(std::string)message); } } @@ -125,8 +125,7 @@ public: #endif private: - - static const std::string log_prefix; + static const char* Prefix(); }; diff --git a/code/XGLLoader.cpp b/code/XGLLoader.cpp index bafcda3f2..df33229a6 100644 --- a/code/XGLLoader.cpp +++ b/code/XGLLoader.cpp @@ -83,8 +83,11 @@ struct free_it }; namespace Assimp { // this has to be in here because LogFunctions is in ::Assimp -template<> const std::string LogFunctions::log_prefix = "XGL: "; - + template<> const char* LogFunctions::Prefix() + { + static auto prefix = "XGL: "; + return prefix; + } } static const aiImporterDesc desc = { From 3803a5181cc1e0e42de26854244db988829b4036 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Fri, 6 Oct 2017 20:32:33 +1100 Subject: [PATCH 237/305] Fixed warnings on MSVC caused by implicit conversions from double to float. --- code/glTF2Asset.inl | 4 ++-- code/glTF2Exporter.cpp | 6 +++--- code/glTFExporter.cpp | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 3082ebfab..8b50fa1d3 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -724,7 +724,7 @@ namespace { SetTextureProperties(r, prop, out); if (Value* scale = FindNumber(*prop, "scale")) { - out.scale = scale->GetDouble(); + out.scale = static_cast(scale->GetDouble()); } } } @@ -735,7 +735,7 @@ namespace { SetTextureProperties(r, prop, out); if (Value* strength = FindNumber(*prop, "strength")) { - out.strength = strength->GetDouble(); + out.strength = static_cast(strength->GetDouble()); } } } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 4ed8f20ff..8f46f7dc7 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -563,7 +563,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref(jointNamesIndex); vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; jointsPerVertex[vertexId] += 1; @@ -872,7 +872,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmNumPositionKeys / numKeyframes; // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. // Check if we have to cast type here. e.g. uint16_t() - timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; + timeData[i] = static_cast(nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond); } Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); @@ -953,7 +953,7 @@ void glTF2Exporter::ExportAnimations() Ref animRef = mAsset->animations.Create(name); // Parameters - ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, static_cast(anim->mTicksPerSecond)); for (unsigned int j = 0; j < 3; ++j) { std::string channelType; diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index a5df09ac7..92abcc15c 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -473,7 +473,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref(jointNamesIndex); vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; jointsPerVertex[vertexId] += 1; @@ -872,7 +872,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmNumPositionKeys / numKeyframes; // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. // Check if we have to cast type here. e.g. uint16_t() - timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; + timeData[i] = static_cast(nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond); } Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); @@ -953,7 +953,7 @@ void glTFExporter::ExportAnimations() Ref animRef = mAsset->animations.Create(name); /******************* Parameters ********************/ - ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, static_cast(anim->mTicksPerSecond)); for (unsigned int j = 0; j < 3; ++j) { std::string channelType; From c1f93a69aeea260c729653b1f50607e825f96cc6 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 6 Oct 2017 15:42:05 +0200 Subject: [PATCH 238/305] Revert "WIP: Enable Travis clang build" --- .travis.sh | 2 +- .travis.yml | 14 - CMakeLists.txt | 13 + code/AMFImporter_Postprocess.cpp | 1 + code/FBXImporter.h | 8 - code/IFCLoader.h | 7 - code/OpenGEXImporter.cpp | 31 +- code/OpenGEXImporter.h | 6 +- code/UnrealLoader.cpp | 4 +- code/glTF2Asset.inl | 4 +- code/glTF2Exporter.cpp | 6 +- code/glTFExporter.cpp | 6 +- .../rapidjson/include/rapidjson/allocators.h | 42 +- .../rapidjson/include/rapidjson/document.h | 1073 ++------- .../include/rapidjson/encodedstream.h | 58 +- .../rapidjson/include/rapidjson/encodings.h | 233 +- .../rapidjson/include/rapidjson/error/en.h | 21 +- .../rapidjson/include/rapidjson/error/error.h | 27 +- .../include/rapidjson/filereadstream.h | 13 +- .../include/rapidjson/filewritestream.h | 17 +- contrib/rapidjson/include/rapidjson/fwd.h | 151 -- .../include/rapidjson/internal/biginteger.h | 14 +- .../include/rapidjson/internal/diyfp.h | 19 +- .../include/rapidjson/internal/dtoa.h | 50 +- .../include/rapidjson/internal/ieee754.h | 3 +- .../include/rapidjson/internal/regex.h | 734 ------ .../include/rapidjson/internal/stack.h | 70 +- .../include/rapidjson/internal/strfunc.h | 32 +- .../include/rapidjson/internal/strtod.h | 39 +- .../include/rapidjson/internal/swap.h | 9 - .../include/rapidjson/istreamwrapper.h | 115 - .../include/rapidjson/memorybuffer.h | 2 +- .../include/rapidjson/memorystream.h | 16 +- .../include/rapidjson/msinttypes/stdint.h | 8 +- .../include/rapidjson/ostreamwrapper.h | 81 - contrib/rapidjson/include/rapidjson/pointer.h | 131 +- .../include/rapidjson/prettywriter.h | 98 +- .../rapidjson/include/rapidjson/rapidjson.h | 314 +-- contrib/rapidjson/include/rapidjson/reader.h | 1345 +++-------- contrib/rapidjson/include/rapidjson/schema.h | 2016 ----------------- contrib/rapidjson/include/rapidjson/stream.h | 179 -- .../include/rapidjson/stringbuffer.h | 30 +- contrib/rapidjson/include/rapidjson/writer.h | 468 +--- contrib/rapidjson/license.txt | 2 +- contrib/rapidjson/readme.md | 81 +- test/unit/TestModelFactory.h | 2 +- test/unit/utMetadata.cpp | 1 - test/unit/utRemoveVCProcess.cpp | 2 +- 48 files changed, 1060 insertions(+), 6538 deletions(-) delete mode 100644 contrib/rapidjson/include/rapidjson/fwd.h delete mode 100644 contrib/rapidjson/include/rapidjson/internal/regex.h delete mode 100644 contrib/rapidjson/include/rapidjson/istreamwrapper.h delete mode 100644 contrib/rapidjson/include/rapidjson/ostreamwrapper.h delete mode 100644 contrib/rapidjson/include/rapidjson/schema.h delete mode 100644 contrib/rapidjson/include/rapidjson/stream.h diff --git a/.travis.sh b/.travis.sh index 1ab1ee2b1..f19c3a000 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_WERROR=ON -DASSIMP_ASAN=$ASAN + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ERROR=ON -DASSIMP_ASAN=$ASAN } if [ $ANDROID ]; then diff --git a/.travis.yml b/.travis.yml index 2eab656cc..d59689f78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,10 +25,6 @@ branches: os: - linux -compiler: - - gcc - - clang - env: global: # COVERITY_SCAN_TOKEN @@ -38,10 +34,6 @@ env: matrix: exclude: - os: linux - compiler: gcc - env: - - os: linux - compiler: clang env: include: @@ -57,12 +49,6 @@ matrix: - os: linux compiler: gcc env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF - - os: linux - compiler: clang - env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON - - os: linux - compiler: clang - env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 0aae76837..5c0fe062e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,19 @@ IF(ASSIMP_DOUBLE_PRECISION) ADD_DEFINITIONS(-DASSIMP_DOUBLE_PRECISION) ENDIF(ASSIMP_DOUBLE_PRECISION) +# Check for OpenMP support +find_package(OpenMP) +if (OPENMP_FOUND) + SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + + IF(MSVC) + IF(MSVC_VERSION GREATER 1910) + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:twoPhase-") + ENDIF() + ENDIF() +endif() + CONFIGURE_FILE( ${CMAKE_CURRENT_LIST_DIR}/revision.h.in ${CMAKE_CURRENT_BINARY_DIR}/revision.h diff --git a/code/AMFImporter_Postprocess.cpp b/code/AMFImporter_Postprocess.cpp index 789a11eb8..085336c51 100644 --- a/code/AMFImporter_Postprocess.cpp +++ b/code/AMFImporter_Postprocess.cpp @@ -686,6 +686,7 @@ std::list mesh_idx; tmesh->mNumVertices = static_cast(vert_arr.size()); tmesh->mVertices = new aiVector3D[tmesh->mNumVertices]; tmesh->mColors[0] = new aiColor4D[tmesh->mNumVertices]; + tmesh->mFaces = new aiFace[face_list_cur.size()]; memcpy(tmesh->mVertices, vert_arr.data(), tmesh->mNumVertices * sizeof(aiVector3D)); memcpy(tmesh->mColors[0], col_arr.data(), tmesh->mNumVertices * sizeof(aiColor4D)); diff --git a/code/FBXImporter.h b/code/FBXImporter.h index 54be8384e..43be97ffa 100644 --- a/code/FBXImporter.h +++ b/code/FBXImporter.h @@ -58,14 +58,6 @@ namespace Formatter { typedef class basic_formatter< char, std::char_traits, std::allocator > format; } -#ifndef _MSC_VER -// GCC and Clang need to see this explicit declaration to avoid warning -// MSVC complains about redeclaration even though this is just the -// declaration, not the definition -class FBXImporter; -template<> const std::string LogFunctions::log_prefix; -#endif // _MSC_VER - // ------------------------------------------------------------------------------------------- /** Load the Autodesk FBX file format. diff --git a/code/IFCLoader.h b/code/IFCLoader.h index 000683bb8..4cf116f8e 100644 --- a/code/IFCLoader.h +++ b/code/IFCLoader.h @@ -60,13 +60,6 @@ namespace Assimp { class DB; } -#ifndef _MSC_VER - // GCC and Clang need to see this explicit declaration to avoid warning - // MSVC complains about redeclaration even though this is just the - // declaration, not the definition - class IFCImporter; - template<> const std::string LogFunctions::log_prefix; -#endif // _MSC_VER // ------------------------------------------------------------------------------------------- /** Load the IFC format, which is an open specification to describe building and construction diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 47e3e3e8c..d9c58e91c 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -307,7 +307,6 @@ void OpenGEXImporter::InternReadFile( const std::string &filename, aiScene *pSce std::vector buffer; TextFileToBuffer( file, buffer ); - pIOHandler->Close( file ); OpenDDLParser myParser; myParser.setBuffer( &buffer[ 0 ], buffer.size() ); @@ -322,7 +321,6 @@ void OpenGEXImporter::InternReadFile( const std::string &filename, aiScene *pSce copyMeshes( pScene ); copyCameras( pScene ); copyLights( pScene ); - copyMaterials( pScene ); resolveReferences(); createNodeTree( pScene ); } @@ -526,7 +524,7 @@ void OpenGEXImporter::handleObjectRefNode( DDLNode *node, aiScene *pScene ) { m_currentNode->mNumMeshes = static_cast(objRefNames.size()); m_currentNode->mMeshes = new unsigned int[ objRefNames.size() ]; if ( !objRefNames.empty() ) { - m_unresolvedRefStack.push_back( std::unique_ptr( new RefInfo( m_currentNode, RefInfo::MeshRef, objRefNames ) ) ); + m_unresolvedRefStack.push_back( new RefInfo( m_currentNode, RefInfo::MeshRef, objRefNames ) ); } } else if ( m_tokenType == Grammar::LightNodeToken ) { // TODO! @@ -545,7 +543,7 @@ void OpenGEXImporter::handleMaterialRefNode( ODDLParser::DDLNode *node, aiScene std::vector matRefNames; getRefNames( node, matRefNames ); if( !matRefNames.empty() ) { - m_unresolvedRefStack.push_back( std::unique_ptr( new RefInfo( m_currentNode, RefInfo::MaterialRef, matRefNames ) ) ); + m_unresolvedRefStack.push_back( new RefInfo( m_currentNode, RefInfo::MaterialRef, matRefNames ) ); } } @@ -1159,19 +1157,6 @@ void OpenGEXImporter::copyLights( aiScene *pScene ) { std::copy( m_lightCache.begin(), m_lightCache.end(), pScene->mLights ); } -//------------------------------------------------------------------------------------------------ -void OpenGEXImporter::copyMaterials( aiScene *pScene ) { - ai_assert( nullptr != pScene ); - - if ( m_materialCache.empty() ) { - return; - } - - pScene->mNumMaterials = static_cast(m_materialCache.size()); - pScene->mMaterials = new aiMaterial*[ pScene->mNumMaterials ]; - std::copy( m_materialCache.begin(), m_materialCache.end(), pScene->mMaterials ); -} - //------------------------------------------------------------------------------------------------ void OpenGEXImporter::resolveReferences() { if( m_unresolvedRefStack.empty() ) { @@ -1179,8 +1164,8 @@ void OpenGEXImporter::resolveReferences() { } RefInfo *currentRefInfo( nullptr ); - for( auto it = m_unresolvedRefStack.begin(); it != m_unresolvedRefStack.end(); ++it ) { - currentRefInfo = it->get(); + for( std::vector::iterator it = m_unresolvedRefStack.begin(); it != m_unresolvedRefStack.end(); ++it ) { + currentRefInfo = *it; if( nullptr != currentRefInfo ) { aiNode *node( currentRefInfo->m_node ); if( RefInfo::MeshRef == currentRefInfo->m_type ) { @@ -1246,9 +1231,9 @@ void OpenGEXImporter::pushNode( aiNode *node, aiScene *pScene ) { if( m_nodeChildMap.end() == it ) { info = new ChildInfo; m_root = info; - m_nodeChildMap[ node->mParent ] = std::unique_ptr(info); + m_nodeChildMap[ node->mParent ] = info; } else { - info = it->second.get(); + info = it->second; } info->m_children.push_back( node ); } else { @@ -1258,9 +1243,9 @@ void OpenGEXImporter::pushNode( aiNode *node, aiScene *pScene ) { NodeChildMap::iterator it( m_nodeChildMap.find( node->mParent ) ); if( m_nodeChildMap.end() == it ) { info = new ChildInfo; - m_nodeChildMap[ node->mParent ] = std::unique_ptr(info); + m_nodeChildMap[ node->mParent ] = info; } else { - info = it->second.get(); + info = it->second; } info->m_children.push_back( node ); } diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index c0cde579c..bb6b45140 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -49,7 +49,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include namespace ODDLParser { class DDLNode; @@ -133,7 +132,6 @@ protected: void copyMeshes( aiScene *pScene ); void copyCameras( aiScene *pScene ); void copyLights( aiScene *pScene ); - void copyMaterials( aiScene *pScene ); void resolveReferences(); void pushNode( aiNode *node, aiScene *pScene ); aiNode *popNode(); @@ -181,7 +179,7 @@ private: std::list m_children; }; ChildInfo *m_root; - typedef std::map > NodeChildMap; + typedef std::map NodeChildMap; NodeChildMap m_nodeChildMap; std::vector m_meshCache; @@ -202,7 +200,7 @@ private: std::vector m_cameraCache; std::vector m_lightCache; std::vector m_nodeStack; - std::vector > m_unresolvedRefStack; + std::vector m_unresolvedRefStack; }; } // Namespace OpenGEX diff --git a/code/UnrealLoader.cpp b/code/UnrealLoader.cpp index a79a2a5c5..c8382cb01 100644 --- a/code/UnrealLoader.cpp +++ b/code/UnrealLoader.cpp @@ -157,7 +157,7 @@ void UnrealImporter::InternReadFile( const std::string& pFile, DefaultLogger::get()->debug("UNREAL: uc file is " + uc_path); // and open the files ... we can't live without them - std::unique_ptr p(pIOHandler->Open(d_path)); + IOStream* p = pIOHandler->Open(d_path); if (!p) throw DeadlyImportError("UNREAL: Unable to open _d file"); StreamReaderLE d_reader(pIOHandler->Open(d_path)); @@ -203,7 +203,7 @@ void UnrealImporter::InternReadFile( const std::string& pFile, d_reader.IncPtr(1); } - p.reset(pIOHandler->Open(a_path)); + p = pIOHandler->Open(a_path); if (!p) throw DeadlyImportError("UNREAL: Unable to open _a file"); StreamReaderLE a_reader(pIOHandler->Open(a_path)); diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 8b50fa1d3..3082ebfab 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -724,7 +724,7 @@ namespace { SetTextureProperties(r, prop, out); if (Value* scale = FindNumber(*prop, "scale")) { - out.scale = static_cast(scale->GetDouble()); + out.scale = scale->GetDouble(); } } } @@ -735,7 +735,7 @@ namespace { SetTextureProperties(r, prop, out); if (Value* strength = FindNumber(*prop, "strength")) { - out.strength = static_cast(strength->GetDouble()); + out.strength = strength->GetDouble(); } } } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 8f46f7dc7..4ed8f20ff 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -563,7 +563,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref(jointNamesIndex); + vertexJointData[vertexId][jointsPerVertex[vertexId]] = jointNamesIndex; vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; jointsPerVertex[vertexId] += 1; @@ -872,7 +872,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmNumPositionKeys / numKeyframes; // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. // Check if we have to cast type here. e.g. uint16_t() - timeData[i] = static_cast(nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond); + timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; } Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); @@ -953,7 +953,7 @@ void glTF2Exporter::ExportAnimations() Ref animRef = mAsset->animations.Create(name); // Parameters - ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, static_cast(anim->mTicksPerSecond)); + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); for (unsigned int j = 0; j < 3; ++j) { std::string channelType; diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index 92abcc15c..a5df09ac7 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -473,7 +473,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref(jointNamesIndex); + vertexJointData[vertexId][jointsPerVertex[vertexId]] = jointNamesIndex; vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; jointsPerVertex[vertexId] += 1; @@ -872,7 +872,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmNumPositionKeys / numKeyframes; // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. // Check if we have to cast type here. e.g. uint16_t() - timeData[i] = static_cast(nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond); + timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; } Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); @@ -953,7 +953,7 @@ void glTFExporter::ExportAnimations() Ref animRef = mAsset->animations.Create(name); /******************* Parameters ********************/ - ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, static_cast(anim->mTicksPerSecond)); + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); for (unsigned int j = 0; j < 3; ++j) { std::string channelType; diff --git a/contrib/rapidjson/include/rapidjson/allocators.h b/contrib/rapidjson/include/rapidjson/allocators.h index 655f4a385..d74a67155 100644 --- a/contrib/rapidjson/include/rapidjson/allocators.h +++ b/contrib/rapidjson/include/rapidjson/allocators.h @@ -179,10 +179,9 @@ public: size = RAPIDJSON_ALIGN(size); if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) - if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) - return NULL; + AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); - void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; + void *buffer = reinterpret_cast(chunkHead_ + 1) + chunkHead_->size; chunkHead_->size += size; return buffer; } @@ -195,16 +194,14 @@ public: if (newSize == 0) return NULL; - originalSize = RAPIDJSON_ALIGN(originalSize); - newSize = RAPIDJSON_ALIGN(newSize); - // Do not shrink if new size is smaller than original if (originalSize >= newSize) return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { + if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { size_t increment = static_cast(newSize - originalSize); + increment = RAPIDJSON_ALIGN(increment); if (chunkHead_->size + increment <= chunkHead_->capacity) { chunkHead_->size += increment; return originalPtr; @@ -212,13 +209,11 @@ public: } // Realloc process: allocate and copy memory, do not free original buffer. - if (void* newBuffer = Malloc(newSize)) { - if (originalSize) - std::memcpy(newBuffer, originalPtr, originalSize); - return newBuffer; - } - else - return NULL; + void* newBuffer = Malloc(newSize); + RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. + if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; } //! Frees a memory block (concept Allocator) @@ -232,20 +227,15 @@ private: //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. - \return true if success. */ - bool AddChunk(size_t capacity) { + void AddChunk(size_t capacity) { if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); - if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; - return true; - } - else - return false; + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); + ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity)); + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; } static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. diff --git a/contrib/rapidjson/include/rapidjson/document.h b/contrib/rapidjson/include/rapidjson/document.h index 93b091f64..c6acbd907 100644 --- a/contrib/rapidjson/include/rapidjson/document.h +++ b/contrib/rapidjson/include/rapidjson/document.h @@ -20,29 +20,40 @@ #include "reader.h" #include "internal/meta.h" #include "internal/strfunc.h" -#include "memorystream.h" -#include "encodedstream.h" #include // placement new -#include -RAPIDJSON_DIAG_PUSH #ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - -#ifdef __GNUC__ +#elif defined(__GNUC__) +RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) -#if __GNUC__ >= 6 -RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions #endif -#endif // __GNUC__ + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_HAS_STDSTRING +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def RAPIDJSON_HAS_STDSTRING + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#endif // !defined(RAPIDJSON_HAS_STDSTRING) + +#if RAPIDJSON_HAS_STDSTRING +#include +#endif // RAPIDJSON_HAS_STDSTRING #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::iterator, std::random_access_iterator_tag @@ -58,9 +69,6 @@ RAPIDJSON_NAMESPACE_BEGIN template class GenericValue; -template -class GenericDocument; - //! Name-value pair in a JSON object value. /*! This class was internal to GenericValue. It used to be a inner struct. @@ -147,7 +155,6 @@ public: Otherwise, the copy constructor is implicitly defined. */ GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} - Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } //! @name stepping //@{ @@ -250,7 +257,6 @@ struct GenericStringRef { typedef CharType Ch; //!< character type of the string //! Create string reference from \c const character array -#ifndef __clang__ // -Wdocumentation /*! This constructor implicitly creates a constant string reference from a \c const character array. It has better performance than @@ -273,13 +279,11 @@ struct GenericStringRef { In such cases, the referenced string should be \b copied to the GenericValue instead. */ -#endif template GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT : s(str), length(N-1) {} //! Explicitly create string reference from \c const character pointer -#ifndef __clang__ // -Wdocumentation /*! This constructor can be used to \b explicitly create a reference to a constant string pointer. @@ -298,23 +302,18 @@ struct GenericStringRef { In such cases, the referenced string should be \b copied to the GenericValue instead. */ -#endif explicit GenericStringRef(const CharType* str) - : s(str), length(NotNullStrLen(str)) {} + : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != NULL); } //! Create constant string reference from pointer and length -#ifndef __clang__ // -Wdocumentation /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \param len length of the string, excluding the trailing NULL terminator \post \ref s == str && \ref length == len \note Constant complexity. */ -#endif GenericStringRef(const CharType* str, SizeType len) - : s(RAPIDJSON_LIKELY(str) ? str : emptyString), length(len) { RAPIDJSON_ASSERT(str != 0 || len == 0u); } - - GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} + : s(str), length(len) { RAPIDJSON_ASSERT(s != NULL); } //! implicit conversion to plain CharType pointer operator const Ch *() const { return s; } @@ -323,24 +322,13 @@ struct GenericStringRef { const SizeType length; //!< length of the string (excluding the trailing NULL terminator) private: - SizeType NotNullStrLen(const CharType* str) { - RAPIDJSON_ASSERT(str != 0); - return internal::StrLen(str); - } - - /// Empty string - used when passing in a NULL pointer - static const Ch emptyString[]; - + //! Disallow copy-assignment + GenericStringRef operator=(const GenericStringRef&); //! Disallow construction from non-const array template GenericStringRef(CharType (&str)[N]) /* = delete */; - //! Copy assignment operator not permitted - immutable type - GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */; }; -template -const CharType GenericStringRef::emptyString[] = { CharType() }; - //! Mark a character pointer as constant string /*! Mark a plain character pointer as a "string literal". This function can be used to avoid copying a character string to be referenced as a @@ -355,7 +343,7 @@ const CharType GenericStringRef::emptyString[] = { CharType() }; */ template inline GenericStringRef StringRef(const CharType* str) { - return GenericStringRef(str); + return GenericStringRef(str, internal::StrLen(str)); } //! Mark a character pointer as constant string @@ -413,127 +401,6 @@ template struct IsGenericValue : IsGenericValueImpl::Type {}; } // namespace internal -/////////////////////////////////////////////////////////////////////////////// -// TypeHelper - -namespace internal { - -template -struct TypeHelper {}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsBool(); } - static bool Get(const ValueType& v) { return v.GetBool(); } - static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } - static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } -}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsInt(); } - static int Get(const ValueType& v) { return v.GetInt(); } - static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } - static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } -}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsUint(); } - static unsigned Get(const ValueType& v) { return v.GetUint(); } - static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } - static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } -}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsInt64(); } - static int64_t Get(const ValueType& v) { return v.GetInt64(); } - static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } - static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } -}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsUint64(); } - static uint64_t Get(const ValueType& v) { return v.GetUint64(); } - static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } - static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } -}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsDouble(); } - static double Get(const ValueType& v) { return v.GetDouble(); } - static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } - static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } -}; - -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsFloat(); } - static float Get(const ValueType& v) { return v.GetFloat(); } - static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } - static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } -}; - -template -struct TypeHelper { - typedef const typename ValueType::Ch* StringType; - static bool Is(const ValueType& v) { return v.IsString(); } - static StringType Get(const ValueType& v) { return v.GetString(); } - static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } - static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } -}; - -#if RAPIDJSON_HAS_STDSTRING -template -struct TypeHelper > { - typedef std::basic_string StringType; - static bool Is(const ValueType& v) { return v.IsString(); } - static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } - static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } -}; -#endif - -template -struct TypeHelper { - typedef typename ValueType::Array ArrayType; - static bool Is(const ValueType& v) { return v.IsArray(); } - static ArrayType Get(ValueType& v) { return v.GetArray(); } - static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } - static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } -}; - -template -struct TypeHelper { - typedef typename ValueType::ConstArray ArrayType; - static bool Is(const ValueType& v) { return v.IsArray(); } - static ArrayType Get(const ValueType& v) { return v.GetArray(); } -}; - -template -struct TypeHelper { - typedef typename ValueType::Object ObjectType; - static bool Is(const ValueType& v) { return v.IsObject(); } - static ObjectType Get(ValueType& v) { return v.GetObject(); } - static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } - static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { return v = data; } -}; - -template -struct TypeHelper { - typedef typename ValueType::ConstObject ObjectType; - static bool Is(const ValueType& v) { return v.IsObject(); } - static ObjectType Get(const ValueType& v) { return v.GetObject(); } -}; - -} // namespace internal - -// Forward declarations -template class GenericArray; -template class GenericObject; - /////////////////////////////////////////////////////////////////////////////// // GenericValue @@ -561,21 +428,17 @@ public: typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. typedef GenericValue ValueType; //!< Value type of itself. - typedef GenericArray Array; - typedef GenericArray ConstArray; - typedef GenericObject Object; - typedef GenericObject ConstObject; //!@name Constructors and destructor. //@{ //! Default constructor creates a null value. - GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } + GenericValue() RAPIDJSON_NOEXCEPT : data_(), flags_(kNullFlag) {} #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 - GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { - rhs.data_.f.flags = kNullFlag; // give up contents + GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_), flags_(rhs.flags_) { + rhs.flags_ = kNullFlag; // give up contents } #endif @@ -583,16 +446,6 @@ private: //! Copy constructor is not permitted. GenericValue(const GenericValue& rhs); -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Moving from a GenericDocument is not permitted. - template - GenericValue(GenericDocument&& rhs); - - //! Move assignment from a GenericDocument is not permitted. - template - GenericValue& operator=(GenericDocument&& rhs); -#endif - public: //! Constructor with JSON value type. @@ -600,13 +453,13 @@ public: \param type Type of the value. \note Default content for number is zero. */ - explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { - static const uint16_t defaultFlags[7] = { + explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_(), flags_() { + static const unsigned defaultFlags[7] = { kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, kNumberAnyFlag }; - RAPIDJSON_ASSERT(type >= kNullType && type <= kNumberType); - data_.f.flags = defaultFlags[type]; + RAPIDJSON_ASSERT(type <= kNumberType); + flags_ = defaultFlags[type]; // Use ShortString to store empty string. if (type == kStringType) @@ -618,50 +471,10 @@ public: \tparam SourceAllocator allocator of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). - \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) \see CopyFrom() */ - template - GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { - switch (rhs.GetType()) { - case kObjectType: { - SizeType count = rhs.data_.o.size; - Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); - const typename GenericValue::Member* rm = rhs.GetMembersPointer(); - for (SizeType i = 0; i < count; i++) { - new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); - new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); - } - data_.f.flags = kObjectFlag; - data_.o.size = data_.o.capacity = count; - SetMembersPointer(lm); - } - break; - case kArrayType: { - SizeType count = rhs.data_.a.size; - GenericValue* le = reinterpret_cast(allocator.Malloc(count * sizeof(GenericValue))); - const GenericValue* re = rhs.GetElementsPointer(); - for (SizeType i = 0; i < count; i++) - new (&le[i]) GenericValue(re[i], allocator, copyConstStrings); - data_.f.flags = kArrayFlag; - data_.a.size = data_.a.capacity = count; - SetElementsPointer(le); - } - break; - case kStringType: - if (rhs.data_.f.flags == kConstStringFlag && !copyConstStrings) { - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast(&rhs.data_); - } - else - SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); - break; - default: - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast(&rhs.data_); - break; - } - } + template< typename SourceAllocator > + GenericValue(const GenericValue& rhs, Allocator & allocator); //! Constructor for boolean value. /*! \param b Boolean value @@ -671,125 +484,96 @@ public: */ #ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen template - explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT #else explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT #endif - : data_() { + : data_(), flags_(b ? kTrueFlag : kFalseFlag) { // safe-guard against failing SFINAE RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); - data_.f.flags = b ? kTrueFlag : kFalseFlag; } //! Constructor for int value. - explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { + explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberIntFlag) { data_.n.i64 = i; - data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; + if (i >= 0) + flags_ |= kUintFlag | kUint64Flag; } //! Constructor for unsigned value. - explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { + explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUintFlag) { data_.n.u64 = u; - data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); + if (!(u & 0x80000000)) + flags_ |= kIntFlag | kInt64Flag; } //! Constructor for int64_t value. - explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { + explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberInt64Flag) { data_.n.i64 = i64; - data_.f.flags = kNumberInt64Flag; if (i64 >= 0) { - data_.f.flags |= kNumberUint64Flag; + flags_ |= kNumberUint64Flag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - data_.f.flags |= kUintFlag; + flags_ |= kUintFlag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; + flags_ |= kIntFlag; } else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; + flags_ |= kIntFlag; } //! Constructor for uint64_t value. - explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { + explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUint64Flag) { data_.n.u64 = u64; - data_.f.flags = kNumberUint64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) - data_.f.flags |= kInt64Flag; + flags_ |= kInt64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - data_.f.flags |= kUintFlag; + flags_ |= kUintFlag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; + flags_ |= kIntFlag; } //! Constructor for double value. - explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } - - //! Constructor for float value. - explicit GenericValue(float f) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = static_cast(f); data_.f.flags = kNumberDoubleFlag; } + explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberDoubleFlag) { data_.n.d = d; } //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } + GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(StringRef(s, length)); } //! Constructor for constant string (i.e. do not make a copy of string) - explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } + explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(s); } //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s, length), allocator); } //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } + GenericValue(const Ch*s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Constructor for copy-string from a string object (i.e. do make a copy of string) /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } + GenericValue(const std::basic_string& s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } #endif - //! Constructor for Array. - /*! - \param a An array obtained by \c GetArray(). - \note \c Array is always pass-by-value. - \note the source array is moved into this value and the sourec array becomes empty. - */ - GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { - a.value_.data_ = Data(); - a.value_.data_.f.flags = kArrayFlag; - } - - //! Constructor for Object. - /*! - \param o An object obtained by \c GetObject(). - \note \c Object is always pass-by-value. - \note the source object is moved into this value and the sourec object becomes empty. - */ - GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { - o.value_.data_ = Data(); - o.value_.data_.f.flags = kObjectFlag; - } - //! Destructor. /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(data_.f.flags) { + switch(flags_) { case kArrayFlag: - { - GenericValue* e = GetElementsPointer(); - for (GenericValue* v = e; v != e + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(e); - } + for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(data_.a.elements); break; case kObjectFlag: for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) m->~Member(); - Allocator::Free(GetMembersPointer()); + Allocator::Free(data_.o.members); break; case kCopyStringFlag: - Allocator::Free(const_cast(GetStringPointer())); + Allocator::Free(const_cast(data_.s.str)); break; default: @@ -854,13 +638,12 @@ public: \tparam SourceAllocator Allocator type of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator to use for copying - \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) */ template - GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { - RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { + RAPIDJSON_ASSERT((void*)this != (void const*)&rhs); this->~GenericValue(); - new (this) GenericValue(rhs, allocator, copyConstStrings); + new (this) GenericValue(rhs, allocator); return *this; } @@ -877,20 +660,6 @@ public: return *this; } - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.value, b.value); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - //! Prepare Value for move semantics /*! \return *this */ GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } @@ -940,7 +709,7 @@ public: else return data_.n.u64 == rhs.data_.n.u64; - default: + default: // kTrueType, kFalseType, kNullType return true; } } @@ -988,58 +757,20 @@ public: //!@name Type //@{ - Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } - bool IsNull() const { return data_.f.flags == kNullFlag; } - bool IsFalse() const { return data_.f.flags == kFalseFlag; } - bool IsTrue() const { return data_.f.flags == kTrueFlag; } - bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } - bool IsObject() const { return data_.f.flags == kObjectFlag; } - bool IsArray() const { return data_.f.flags == kArrayFlag; } - bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } - bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } - bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } - bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } - bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } - bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } - bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } - - // Checks whether a number can be losslessly converted to a double. - bool IsLosslessDouble() const { - if (!IsNumber()) return false; - if (IsUint64()) { - uint64_t u = GetUint64(); - volatile double d = static_cast(u); - return (d >= 0.0) - && (d < static_cast((std::numeric_limits::max)())) - && (u == static_cast(d)); - } - if (IsInt64()) { - int64_t i = GetInt64(); - volatile double d = static_cast(i); - return (d >= static_cast((std::numeric_limits::min)())) - && (d < static_cast((std::numeric_limits::max)())) - && (i == static_cast(d)); - } - return true; // double, int, uint are always lossless - } - - // Checks whether a number is a float (possible lossy). - bool IsFloat() const { - if ((data_.f.flags & kDoubleFlag) == 0) - return false; - double d = GetDouble(); - return d >= -3.4028234e38 && d <= 3.4028234e38; - } - // Checks whether a number can be losslessly converted to a float. - bool IsLosslessFloat() const { - if (!IsNumber()) return false; - double a = GetDouble(); - if (a < static_cast(-(std::numeric_limits::max)()) - || a > static_cast((std::numeric_limits::max)())) - return false; - double b = static_cast(static_cast(a)); - return a >= b && a <= b; // Prevent -Wfloat-equal - } + Type GetType() const { return static_cast(flags_ & kTypeMask); } + bool IsNull() const { return flags_ == kNullFlag; } + bool IsFalse() const { return flags_ == kFalseFlag; } + bool IsTrue() const { return flags_ == kTrueFlag; } + bool IsBool() const { return (flags_ & kBoolFlag) != 0; } + bool IsObject() const { return flags_ == kObjectFlag; } + bool IsArray() const { return flags_ == kArrayFlag; } + bool IsNumber() const { return (flags_ & kNumberFlag) != 0; } + bool IsInt() const { return (flags_ & kIntFlag) != 0; } + bool IsUint() const { return (flags_ & kUintFlag) != 0; } + bool IsInt64() const { return (flags_ & kInt64Flag) != 0; } + bool IsUint64() const { return (flags_ & kUint64Flag) != 0; } + bool IsDouble() const { return (flags_ & kDoubleFlag) != 0; } + bool IsString() const { return (flags_ & kStringFlag) != 0; } //@} @@ -1053,7 +784,7 @@ public: //!@name Bool //@{ - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return flags_ == kTrueFlag; } //!< Set boolean value /*! \post IsBool() == true */ GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } @@ -1106,14 +837,8 @@ public: return member->value; else { RAPIDJSON_ASSERT(false); // see above note - - // This will generate -Wexit-time-destructors in clang - // static GenericValue NullValue; - // return NullValue; - - // Use static buffer and placement-new to prevent destruction - static char buffer[sizeof(GenericValue)]; - return *new (buffer) GenericValue(); + static GenericValue NullValue; + return NullValue; } } template @@ -1127,16 +852,16 @@ public: //! Const member iterator /*! \pre IsObject() == true */ - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members); } //! Const \em past-the-end member iterator /*! \pre IsObject() == true */ - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members + data_.o.size); } //! Member iterator /*! \pre IsObject() == true */ - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members); } //! \em Past-the-end member iterator /*! \pre IsObject() == true */ - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members + data_.o.size); } //! Check whether a member exists in the object. /*! @@ -1224,8 +949,8 @@ public: \return Iterator to member, if it exists. Otherwise returns \ref MemberEnd(). */ - MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } - ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } + MemberIterator FindMember(const std::basic_string& name) { return FindMember(StringRef(name)); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(StringRef(name)); } #endif //! Add a member (name-value pair) to the object. @@ -1242,21 +967,20 @@ public: RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - ObjectData& o = data_.o; + Object& o = data_.o; if (o.size >= o.capacity) { if (o.capacity == 0) { o.capacity = kDefaultObjectCapacity; - SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); + o.members = reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member))); } else { SizeType oldCapacity = o.capacity; o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); + o.members = reinterpret_cast(allocator.Realloc(o.members, oldCapacity * sizeof(Member), o.capacity * sizeof(Member))); } } - Member* members = GetMembersPointer(); - members[o.size].name.RawAssign(name); - members[o.size].value.RawAssign(value); + o.members[o.size].name.RawAssign(name); + o.members[o.size].value.RawAssign(value); o.size++; return *this; } @@ -1435,14 +1159,18 @@ public: MemberIterator RemoveMember(MemberIterator m) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(data_.o.members != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) - *m = *last; // Move the last one to this place - else - m->~Member(); // Only one left, just destroy + MemberIterator last(data_.o.members + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) { + // Move the last one to this place + *m = *last; + } + else { + // Only one left, just destroy + m->~Member(); + } --data_.o.size; return m; } @@ -1472,7 +1200,7 @@ public: MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(data_.o.members != 0); RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); @@ -1480,39 +1208,11 @@ public: MemberIterator pos = MemberBegin() + (first - MemberBegin()); for (MemberIterator itr = pos; itr != last; ++itr) itr->~Member(); - std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); - data_.o.size -= static_cast(last - first); + std::memmove(&*pos, &*last, (MemberEnd() - last) * sizeof(Member)); + data_.o.size -= (last - first); return pos; } - //! Erase a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note Linear time complexity. - */ - bool EraseMember(const Ch* name) { - GenericValue n(StringRef(name)); - return EraseMember(n); - } - -#if RAPIDJSON_HAS_STDSTRING - bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } -#endif - - template - bool EraseMember(const GenericValue& name) { - MemberIterator m = FindMember(name); - if (m != MemberEnd()) { - EraseMember(m); - return true; - } - else - return false; - } - - Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } - ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } - //@} //!@name Array @@ -1520,7 +1220,7 @@ public: //! Set this value as an empty array. /*! \post IsArray == true */ - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } //! Get the number of elements in array. SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } @@ -1537,9 +1237,8 @@ public: */ void Clear() { RAPIDJSON_ASSERT(IsArray()); - GenericValue* e = GetElementsPointer(); - for (GenericValue* v = e; v != e + data_.a.size; ++v) - v->~GenericValue(); + for (SizeType i = 0; i < data_.a.size; ++i) + data_.a.elements[i].~GenericValue(); data_.a.size = 0; } @@ -1551,16 +1250,16 @@ public: GenericValue& operator[](SizeType index) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(index < data_.a.size); - return GetElementsPointer()[index]; + return data_.a.elements[index]; } const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } //! Element iterator /*! \pre IsArray() == true */ - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements; } //! \em Past-the-end element iterator /*! \pre IsArray() == true */ - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements + data_.a.size; } //! Constant element iterator /*! \pre IsArray() == true */ ConstValueIterator Begin() const { return const_cast(*this).Begin(); } @@ -1577,7 +1276,7 @@ public: GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsArray()); if (newCapacity > data_.a.capacity) { - SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); + data_.a.elements = (GenericValue*)allocator.Realloc(data_.a.elements, data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)); data_.a.capacity = newCapacity; } return *this; @@ -1597,7 +1296,7 @@ public: RAPIDJSON_ASSERT(IsArray()); if (data_.a.size >= data_.a.capacity) Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); - GetElementsPointer()[data_.a.size++].RawAssign(value); + data_.a.elements[data_.a.size++].RawAssign(value); return *this; } @@ -1651,7 +1350,7 @@ public: GenericValue& PopBack() { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(!Empty()); - GetElementsPointer()[--data_.a.size].~GenericValue(); + data_.a.elements[--data_.a.size].~GenericValue(); return *this; } @@ -1677,48 +1376,35 @@ public: ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(data_.a.size > 0); - RAPIDJSON_ASSERT(GetElementsPointer() != 0); + RAPIDJSON_ASSERT(data_.a.elements != 0); RAPIDJSON_ASSERT(first >= Begin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= End()); ValueIterator pos = Begin() + (first - Begin()); for (ValueIterator itr = pos; itr != last; ++itr) itr->~GenericValue(); - std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); - data_.a.size -= static_cast(last - first); + std::memmove(pos, last, (End() - last) * sizeof(GenericValue)); + data_.a.size -= (last - first); return pos; } - Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } - ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } - //@} //!@name Number //@{ - int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + int GetInt() const { RAPIDJSON_ASSERT(flags_ & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(flags_ & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(flags_ & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(flags_ & kUint64Flag); return data_.n.u64; } - //! Get the value as double type. - /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. - */ double GetDouble() const { RAPIDJSON_ASSERT(IsNumber()); - if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) - } - - //! Get the value as float type. - /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. - */ - float GetFloat() const { - return static_cast(GetDouble()); + if ((flags_ & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((flags_ & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((flags_ & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((flags_ & kInt64Flag) != 0) return (double)data_.n.i64; // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((flags_ & kUint64Flag) != 0); return (double)data_.n.u64; // uint64_t -> double (may lose precision) } GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } @@ -1726,19 +1412,18 @@ public: GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } - GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(static_cast(f)); return *this; } //@} //!@name String //@{ - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? data_.ss.str : data_.s.str); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. @@ -1765,7 +1450,7 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { return SetString(StringRef(s, length), allocator); } + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } //! Set this value as a string by copying from source string. /*! \param s source string. @@ -1773,15 +1458,7 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(StringRef(s), allocator); } - - //! Set this value as a string by copying from source string. - /*! \param s source string reference - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s.s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(StringRefType s, Allocator& allocator) { this->~GenericValue(); SetStringRaw(s, allocator); return *this; } + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Set this value as a string by copying from source string. @@ -1791,35 +1468,11 @@ public: \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(StringRef(s), allocator); } + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } #endif //@} - //!@name Array - //@{ - - //! Templated version for checking whether this value is type T. - /*! - \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string - */ - template - bool Is() const { return internal::TypeHelper::Is(*this); } - - template - T Get() const { return internal::TypeHelper::Get(*this); } - - template - T Get() { return internal::TypeHelper::Get(*this); } - - template - ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } - - template - ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } - - //@} - //! Generate events of this value to a Handler. /*! This function adopts the GoF visitor pattern. Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. @@ -1835,35 +1488,35 @@ public: case kTrueType: return handler.Bool(true); case kObjectType: - if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + if (!handler.StartObject()) return false; for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. - if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) + if (!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.flags_ & kCopyFlag) != 0)) return false; - if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) + if (!m->value.Accept(handler)) return false; } return handler.EndObject(data_.o.size); case kArrayType: - if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + if (!handler.StartArray()) return false; - for (const GenericValue* v = Begin(); v != End(); ++v) - if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) + for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) + if (!v->Accept(handler)) return false; return handler.EndArray(data_.a.size); case kStringType: - return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); + return handler.String(GetString(), GetStringLength(), (flags_ & kCopyFlag) != 0); default: RAPIDJSON_ASSERT(GetType() == kNumberType); - if (IsDouble()) return handler.Double(data_.n.d); - else if (IsInt()) return handler.Int(data_.n.i.i); + if (IsInt()) return handler.Int(data_.n.i.i); else if (IsUint()) return handler.Uint(data_.n.u.u); else if (IsInt64()) return handler.Int64(data_.n.i64); - else return handler.Uint64(data_.n.u64); + else if (IsUint64()) return handler.Uint64(data_.n.u64); + else return handler.Double(data_.n.d); } } @@ -1872,16 +1525,16 @@ private: template friend class GenericDocument; enum { - kBoolFlag = 0x0008, - kNumberFlag = 0x0010, - kIntFlag = 0x0020, - kUintFlag = 0x0040, - kInt64Flag = 0x0080, - kUint64Flag = 0x0100, - kDoubleFlag = 0x0200, - kStringFlag = 0x0400, - kCopyFlag = 0x0800, - kInlineStrFlag = 0x1000, + kBoolFlag = 0x100, + kNumberFlag = 0x200, + kIntFlag = 0x400, + kUintFlag = 0x800, + kInt64Flag = 0x1000, + kUint64Flag = 0x2000, + kDoubleFlag = 0x4000, + kStringFlag = 0x100000, + kCopyFlag = 0x200000, + kInlineStrFlag = 0x400000, // Initial flags of different types. kNullFlag = kNullType, @@ -1899,27 +1552,16 @@ private: kObjectFlag = kObjectType, kArrayFlag = kArrayType, - kTypeMask = 0x07 + kTypeMask = 0xFF // bitwise-and with mask of 0xFF can be optimized by compiler }; static const SizeType kDefaultArrayCapacity = 16; static const SizeType kDefaultObjectCapacity = 16; - struct Flag { -#if RAPIDJSON_48BITPOINTER_OPTIMIZATION - char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer -#elif RAPIDJSON_64BIT - char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes -#else - char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes -#endif - uint16_t flags; - }; - struct String { - SizeType length; - SizeType hashcode; //!< reserved const Ch* str; + SizeType length; + unsigned hashcode; //!< reserved }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars @@ -1928,15 +1570,15 @@ private: // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as // the string terminator as well. For getting the string length back from that value just use // "MaxSize - str[LenPos]". - // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, - // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). + // This allows to store 11-chars strings in 32-bit mode and 15-chars strings in 64-bit mode + // inline (for `UTF8`-encoded strings). struct ShortString { - enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + enum { MaxChars = sizeof(String) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; Ch str[MaxChars]; - inline static bool Usable(SizeType len) { return (MaxSize >= len); } - inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } - inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = (Ch)(MaxSize - len); } + inline SizeType GetLength() const { return (SizeType)(MaxSize - str[LenPos]); } }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // By using proper binary layout, retrieval of different integer types do not need conversions. @@ -1965,79 +1607,69 @@ private: double d; }; // 8 bytes - struct ObjectData { + struct Object { + Member* members; SizeType size; SizeType capacity; - Member* members; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - struct ArrayData { + struct Array { + GenericValue* elements; SizeType size; SizeType capacity; - GenericValue* elements; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode union Data { String s; ShortString ss; Number n; - ObjectData o; - ArrayData a; - Flag f; - }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION - - RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } - RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } - RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } - RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } - RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } - RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } + Object o; + Array a; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { - data_.f.flags = kArrayFlag; + flags_ = kArrayFlag; if (count) { - GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); - SetElementsPointer(e); - std::memcpy(e, values, count * sizeof(GenericValue)); + data_.a.elements = (GenericValue*)allocator.Malloc(count * sizeof(GenericValue)); + std::memcpy(data_.a.elements, values, count * sizeof(GenericValue)); } else - SetElementsPointer(0); + data_.a.elements = NULL; data_.a.size = data_.a.capacity = count; } //! Initialize this value as object with initial data, without calling destructor. void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { - data_.f.flags = kObjectFlag; + flags_ = kObjectFlag; if (count) { - Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); - SetMembersPointer(m); - std::memcpy(m, members, count * sizeof(Member)); + data_.o.members = (Member*)allocator.Malloc(count * sizeof(Member)); + std::memcpy(data_.o.members, members, count * sizeof(Member)); } else - SetMembersPointer(0); + data_.o.members = NULL; data_.o.size = data_.o.capacity = count; } //! Initialize this value as constant string, without calling destructor. void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { - data_.f.flags = kConstStringFlag; - SetStringPointer(s); + flags_ = kConstStringFlag; + data_.s.str = s; data_.s.length = s.length; } //! Initialize this value as copy string with initial data, without calling destructor. void SetStringRaw(StringRefType s, Allocator& allocator) { - Ch* str = 0; - if (ShortString::Usable(s.length)) { - data_.f.flags = kShortStringFlag; + Ch* str = NULL; + if(ShortString::Usable(s.length)) { + flags_ = kShortStringFlag; data_.ss.SetLength(s.length); str = data_.ss.str; } else { - data_.f.flags = kCopyStringFlag; + flags_ = kCopyStringFlag; data_.s.length = s.length; - str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); - SetStringPointer(str); + str = (Ch *)allocator.Malloc((s.length + 1) * sizeof(Ch)); + data_.s.str = str; } std::memcpy(str, s, s.length * sizeof(Ch)); str[s.length] = '\0'; @@ -2046,8 +1678,8 @@ private: //! Assignment without calling destructor void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { data_ = rhs.data_; - // data_.f.flags = rhs.data_.f.flags; - rhs.data_.f.flags = kNullFlag; + flags_ = rhs.flags_; + rhs.flags_ = kNullFlag; } template @@ -2067,6 +1699,7 @@ private: } Data data_; + unsigned flags_; }; //! GenericValue with UTF8 encoding @@ -2091,22 +1724,7 @@ public: typedef Allocator AllocatorType; //!< Allocator type from template parameter. //! Constructor - /*! Creates an empty document of specified type. - \param type Mandatory type of object to create. - \param allocator Optional allocator for allocating memory. - \param stackCapacity Optional initial capacity of stack in bytes. - \param stackAllocator Optional allocator for allocating memory for stack. - */ - explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : - GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - } - - //! Constructor - /*! Creates an empty document which type is Null. - \param allocator Optional allocator for allocating memory. + /*! \param allocator Optional allocator for allocating memory. \param stackCapacity Optional initial capacity of stack in bytes. \param stackAllocator Optional allocator for allocating memory for stack. */ @@ -2114,13 +1732,13 @@ public: allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document + : ValueType(std::move(rhs)), allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), stack_(std::move(rhs.stack_)), @@ -2160,54 +1778,6 @@ public: } #endif - //! Exchange the contents of this document with those of another. - /*! - \param rhs Another document. - \note Constant complexity. - \see GenericValue::Swap - */ - GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { - ValueType::Swap(rhs); - stack_.Swap(rhs.stack_); - internal::Swap(allocator_, rhs.allocator_); - internal::Swap(ownAllocator_, rhs.ownAllocator_); - internal::Swap(parseResult_, rhs.parseResult_); - return *this; - } - - // Allow Swap with ValueType. - // Refer to Effective C++ 3rd Edition/Item 33: Avoid hiding inherited names. - using ValueType::Swap; - - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.doc, b.doc); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - - //! Populate this document by a generator which produces SAX events. - /*! \tparam Generator A functor with bool f(Handler) prototype. - \param g Generator functor which sends SAX events to the parameter. - \return The document itself for fluent API. - */ - template - GenericDocument& Populate(Generator& g) { - ClearStackOnExit scope(*this); - if (g(*this)) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document - } - return *this; - } - //!@name Parse from stream //!@{ @@ -2220,13 +1790,13 @@ public: */ template GenericDocument& ParseStream(InputStream& is) { - GenericReader reader( - stack_.HasAllocator() ? &stack_.GetAllocator() : 0); + ValueType::SetNull(); // Remove existing root if exist + GenericReader reader(&stack_.GetAllocator()); ClearStackOnExit scope(*this); parseResult_ = reader.template Parse(is, *this); if (parseResult_) { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + this->RawAssign(*stack_.template Pop(1)); // Add this-> to prevent issue 13. } return *this; } @@ -2285,7 +1855,7 @@ public: \param str Read-only zero-terminated string to be parsed. */ template - GenericDocument& Parse(const typename SourceEncoding::Ch* str) { + GenericDocument& Parse(const Ch* str) { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); GenericStringStream s(str); return ParseStream(s); @@ -2306,42 +1876,6 @@ public: GenericDocument& Parse(const Ch* str) { return Parse(str); } - - template - GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - MemoryStream ms(reinterpret_cast(str), length * sizeof(typename SourceEncoding::Ch)); - EncodedInputStream is(ms); - ParseStream(is); - return *this; - } - - template - GenericDocument& Parse(const Ch* str, size_t length) { - return Parse(str, length); - } - - GenericDocument& Parse(const Ch* str, size_t length) { - return Parse(str, length); - } - -#if RAPIDJSON_HAS_STDSTRING - template - GenericDocument& Parse(const std::basic_string& str) { - // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) - return Parse(str.c_str()); - } - - template - GenericDocument& Parse(const std::basic_string& str) { - return Parse(str.c_str()); - } - - GenericDocument& Parse(const std::basic_string& str) { - return Parse(str); - } -#endif // RAPIDJSON_HAS_STDSTRING - //!@} //!@name Handling parse errors @@ -2356,26 +1890,10 @@ public: //! Get the position of last parsing error in input, 0 otherwise. size_t GetErrorOffset() const { return parseResult_.Offset(); } - //! Implicit conversion to get the last parse result -#ifndef __clang // -Wdocumentation - /*! \return \ref ParseResult of the last parse operation - - \code - Document doc; - ParseResult ok = doc.Parse(json); - if (!ok) - printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); - \endcode - */ -#endif - operator ParseResult() const { return parseResult_; } //!@} //! Get the allocator of this document. - Allocator& GetAllocator() { - RAPIDJSON_ASSERT(allocator_); - return *allocator_; - } + Allocator& GetAllocator() { return *allocator_; } //! Get the capacity of stack in bytes. size_t GetStackCapacity() const { return stack_.GetCapacity(); } @@ -2392,10 +1910,9 @@ private: }; // callers of the following private Handler functions - // template friend class GenericReader; // for parsing + template friend class GenericReader; // for parsing template friend class GenericValue; // for deep copying -public: // Implementation of Handler bool Null() { new (stack_.template Push()) ValueType(); return true; } bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } @@ -2405,14 +1922,6 @@ public: bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } - bool RawNumber(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push()) ValueType(str, length); - return true; - } - bool String(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); @@ -2427,7 +1936,7 @@ public: bool EndObject(SizeType memberCount) { typename ValueType::Member* members = stack_.template Pop(memberCount); - stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); + stack_.template Top()->SetObjectRaw(members, (SizeType)memberCount, GetAllocator()); return true; } @@ -2468,146 +1977,38 @@ private: //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; -//! Helper class for accessing Value of array type. -/*! - Instance of this helper class is obtained by \c GenericValue::GetArray(). - In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. -*/ -template -class GenericArray { -public: - typedef GenericArray ConstArray; - typedef GenericArray Array; - typedef ValueT PlainType; - typedef typename internal::MaybeAddConst::Type ValueType; - typedef ValueType* ValueIterator; // This may be const or non-const iterator - typedef const ValueT* ConstValueIterator; - typedef typename ValueType::AllocatorType AllocatorType; - typedef typename ValueType::StringRefType StringRefType; - - template - friend class GenericValue; - - GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} - GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } - ~GenericArray() {} - - SizeType Size() const { return value_.Size(); } - SizeType Capacity() const { return value_.Capacity(); } - bool Empty() const { return value_.Empty(); } - void Clear() const { value_.Clear(); } - ValueType& operator[](SizeType index) const { return value_[index]; } - ValueIterator Begin() const { return value_.Begin(); } - ValueIterator End() const { return value_.End(); } - GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } - GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } - GenericArray PopBack() const { value_.PopBack(); return *this; } - ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } - ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } - -#if RAPIDJSON_HAS_CXX11_RANGE_FOR - ValueIterator begin() const { return value_.Begin(); } - ValueIterator end() const { return value_.End(); } -#endif - -private: - GenericArray(); - GenericArray(ValueType& value) : value_(value) {} - ValueType& value_; -}; - -//! Helper class for accessing Value of object type. -/*! - Instance of this helper class is obtained by \c GenericValue::GetObject(). - In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. -*/ -template -class GenericObject { -public: - typedef GenericObject ConstObject; - typedef GenericObject Object; - typedef ValueT PlainType; - typedef typename internal::MaybeAddConst::Type ValueType; - typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator - typedef GenericMemberIterator ConstMemberIterator; - typedef typename ValueType::AllocatorType AllocatorType; - typedef typename ValueType::StringRefType StringRefType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename ValueType::Ch Ch; - - template - friend class GenericValue; - - GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} - GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } - ~GenericObject() {} - - SizeType MemberCount() const { return value_.MemberCount(); } - bool ObjectEmpty() const { return value_.ObjectEmpty(); } - template ValueType& operator[](T* name) const { return value_[name]; } - template ValueType& operator[](const GenericValue& name) const { return value_[name]; } -#if RAPIDJSON_HAS_STDSTRING - ValueType& operator[](const std::basic_string& name) const { return value_[name]; } -#endif - MemberIterator MemberBegin() const { return value_.MemberBegin(); } - MemberIterator MemberEnd() const { return value_.MemberEnd(); } - bool HasMember(const Ch* name) const { return value_.HasMember(name); } -#if RAPIDJSON_HAS_STDSTRING - bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } -#endif - template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } - MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } - template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } -#if RAPIDJSON_HAS_STDSTRING - MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } -#endif - GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#if RAPIDJSON_HAS_STDSTRING - GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#endif - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - void RemoveAllMembers() { value_.RemoveAllMembers(); } - bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } -#if RAPIDJSON_HAS_STDSTRING - bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } -#endif - template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } - MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } - MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } - MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } - bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } -#if RAPIDJSON_HAS_STDSTRING - bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } -#endif - template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } - -#if RAPIDJSON_HAS_CXX11_RANGE_FOR - MemberIterator begin() const { return value_.MemberBegin(); } - MemberIterator end() const { return value_.MemberEnd(); } -#endif - -private: - GenericObject(); - GenericObject(ValueType& value) : value_(value) {} - ValueType& value_; -}; +// defined here due to the dependency on GenericDocument +template +template +inline +GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) +{ + switch (rhs.GetType()) { + case kObjectType: + case kArrayType: { // perform deep copy via SAX Handler + GenericDocument d(&allocator); + rhs.Accept(d); + RawAssign(*d.stack_.template Pop(1)); + } + break; + case kStringType: + if (rhs.flags_ == kConstStringFlag) { + flags_ = rhs.flags_; + data_ = *reinterpret_cast(&rhs.data_); + } else { + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + } + break; + default: // kNumberType, kTrueType, kFalseType, kNullType + flags_ = rhs.flags_; + data_ = *reinterpret_cast(&rhs.data_); + } +} RAPIDJSON_NAMESPACE_END + +#if defined(_MSC_VER) || defined(__GNUC__) RAPIDJSON_DIAG_POP +#endif #endif // RAPIDJSON_DOCUMENT_H_ diff --git a/contrib/rapidjson/include/rapidjson/encodedstream.h b/contrib/rapidjson/include/rapidjson/encodedstream.h index 223601c05..7c8863fee 100644 --- a/contrib/rapidjson/include/rapidjson/encodedstream.h +++ b/contrib/rapidjson/include/rapidjson/encodedstream.h @@ -15,19 +15,13 @@ #ifndef RAPIDJSON_ENCODEDSTREAM_H_ #define RAPIDJSON_ENCODEDSTREAM_H_ -#include "stream.h" -#include "memorystream.h" +#include "rapidjson.h" #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - RAPIDJSON_NAMESPACE_BEGIN //! Input byte stream wrapper with a statically bound encoding. @@ -63,38 +57,10 @@ private: Ch current_; }; -//! Specialized for UTF8 MemoryStream. -template <> -class EncodedInputStream, MemoryStream> { -public: - typedef UTF8<>::Ch Ch; - - EncodedInputStream(MemoryStream& is) : is_(is) { - if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); - if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); - if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); - } - Ch Peek() const { return is_.Peek(); } - Ch Take() { return is_.Take(); } - size_t Tell() const { return is_.Tell(); } - - // Not implemented - void Put(Ch) {} - void Flush() {} - Ch* PutBegin() { return 0; } - size_t PutEnd(Ch*) { return 0; } - - MemoryStream& is_; - -private: - EncodedInputStream(const EncodedInputStream&); - EncodedInputStream& operator=(const EncodedInputStream&); -}; - //! Output byte stream wrapper with statically bound encoding. /*! \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. + \tparam InputByteStream Type of input byte stream. For example, FileWriteStream. */ template class EncodedOutputStream { @@ -111,8 +77,8 @@ public: void Flush() { os_.Flush(); } // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} - Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + Ch Peek() const { RAPIDJSON_ASSERT(false); } + Ch Take() { RAPIDJSON_ASSERT(false); } size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } @@ -176,11 +142,11 @@ private: // FF FE UTF-16LE // EF BB BF UTF-8 - const unsigned char* c = reinterpret_cast(is_->Peek4()); + const unsigned char* c = (const unsigned char *)is_->Peek4(); if (!c) return; - unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); + unsigned bom = c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24); hasBOM_ = false; if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } @@ -200,7 +166,7 @@ private: // xx xx xx xx UTF-8 if (!hasBOM_) { - int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); switch (pattern) { case 0x08: type_ = kUTF32BE; break; case 0x0A: type_ = kUTF16BE; break; @@ -227,7 +193,7 @@ private: //! Output stream wrapper with dynamically bound encoding and automatic encoding detection. /*! \tparam CharType Type of character for writing. - \tparam OutputByteStream type of output byte stream to be wrapped. + \tparam InputByteStream type of output byte stream to be wrapped. */ template class AutoUTFOutputStream { @@ -261,8 +227,8 @@ public: void Flush() { os_->Flush(); } // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} - Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + Ch Peek() const { RAPIDJSON_ASSERT(false); } + Ch Take() { RAPIDJSON_ASSERT(false); } size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } @@ -288,10 +254,6 @@ private: RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/encodings.h b/contrib/rapidjson/include/rapidjson/encodings.h index 0df1c3435..90b46ed32 100644 --- a/contrib/rapidjson/include/rapidjson/encodings.h +++ b/contrib/rapidjson/include/rapidjson/encodings.h @@ -120,45 +120,19 @@ struct UTF8 { } } - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - if (codepoint <= 0x7F) - PutUnsafe(os, static_cast(codepoint & 0xFF)); - else if (codepoint <= 0x7FF) { - PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); - PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); - } - else if (codepoint <= 0xFFFF) { - PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); - PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); - PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); - } - } - template static bool Decode(InputStream& is, unsigned* codepoint) { -#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) -#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | ((unsigned char)c & 0x3Fu) +#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) - typename InputStream::Ch c = is.Take(); + Ch c = is.Take(); if (!(c & 0x80)) { - *codepoint = static_cast(c); + *codepoint = (unsigned char)c; return true; } - unsigned char type = GetRange(static_cast(c)); - if (type >= 32) { - *codepoint = 0; - } else { - *codepoint = (0xFFu >> type) & static_cast(c); - } + unsigned char type = GetRange((unsigned char)c); + *codepoint = (0xFF >> type) & (unsigned char)c; bool result = true; switch (type) { case 2: TAIL(); return result; @@ -178,7 +152,7 @@ struct UTF8 { template static bool Validate(InputStream& is, OutputStream& os) { #define COPY() os.Put(c = is.Take()) -#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) Ch c; COPY(); @@ -186,7 +160,7 @@ struct UTF8 { return true; bool result = true; - switch (GetRange(static_cast(c))) { + switch (GetRange((unsigned char)c)) { case 2: TAIL(); return result; case 3: TAIL(); TAIL(); return result; case 4: COPY(); TRANS(0x50); TAIL(); return result; @@ -222,12 +196,12 @@ struct UTF8 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - typename InputByteStream::Ch c = Take(is); - if (static_cast(c) != 0xEFu) return c; + Ch c = Take(is); + if ((unsigned char)c != 0xEFu) return c; c = is.Take(); - if (static_cast(c) != 0xBBu) return c; + if ((unsigned char)c != 0xBBu) return c; c = is.Take(); - if (static_cast(c) != 0xBFu) return c; + if ((unsigned char)c != 0xBFu) return c; c = is.Take(); return c; } @@ -235,15 +209,13 @@ struct UTF8 { template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return static_cast(is.Take()); + return is.Take(); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xEFu)); - os.Put(static_cast(0xBBu)); - os.Put(static_cast(0xBFu)); + os.Put(0xEFu); os.Put(0xBBu); os.Put(0xBFu); } template @@ -283,38 +255,22 @@ struct UTF16 { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; os.Put(static_cast((v >> 10) | 0xD800)); - os.Put(static_cast((v & 0x3FF) | 0xDC00)); - } - } - - - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - PutUnsafe(os, static_cast(codepoint)); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - PutUnsafe(os, static_cast((v >> 10) | 0xD800)); - PutUnsafe(os, static_cast((v & 0x3FF) | 0xDC00)); + os.Put((v & 0x3FF) | 0xDC00); } } template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - typename InputStream::Ch c = is.Take(); + Ch c = is.Take(); if (c < 0xD800 || c > 0xDFFF) { - *codepoint = static_cast(c); + *codepoint = c; return true; } else if (c <= 0xDBFF) { - *codepoint = (static_cast(c) & 0x3FF) << 10; + *codepoint = (c & 0x3FF) << 10; c = is.Take(); - *codepoint |= (static_cast(c) & 0x3FF); + *codepoint |= (c & 0x3FF); *codepoint += 0x10000; return c >= 0xDC00 && c <= 0xDFFF; } @@ -325,8 +281,8 @@ struct UTF16 { static bool Validate(InputStream& is, OutputStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - typename InputStream::Ch c; - os.Put(static_cast(c = is.Take())); + Ch c; + os.Put(c = is.Take()); if (c < 0xD800 || c > 0xDFFF) return true; else if (c <= 0xDBFF) { @@ -344,29 +300,28 @@ struct UTF16LE : UTF16 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return static_cast(c) == 0xFEFFu ? Take(is) : c; + return (unsigned short)c == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(is.Take()); - c |= static_cast(static_cast(is.Take())) << 8; - return static_cast(c); + CharType c = (unsigned char)is.Take(); + c |= (unsigned char)is.Take() << 8; + return c; } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xFFu)); - os.Put(static_cast(0xFEu)); + os.Put(0xFFu); os.Put(0xFEu); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(static_cast(c) & 0xFFu)); - os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(c & 0xFFu); + os.Put((c >> 8) & 0xFFu); } }; @@ -377,29 +332,28 @@ struct UTF16BE : UTF16 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return static_cast(c) == 0xFEFFu ? Take(is) : c; + return (unsigned short)c == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(static_cast(is.Take())) << 8; - c |= static_cast(is.Take()); - return static_cast(c); + CharType c = (unsigned char)is.Take() << 8; + c |= (unsigned char)is.Take(); + return c; } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xFEu)); - os.Put(static_cast(0xFFu)); + os.Put(0xFEu); os.Put(0xFFu); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); - os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put((c >> 8) & 0xFFu); + os.Put(c & 0xFFu); } }; @@ -428,13 +382,6 @@ struct UTF32 { os.Put(codepoint); } - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - PutUnsafe(os, codepoint); - } - template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); @@ -459,35 +406,32 @@ struct UTF32LE : UTF32 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + return (unsigned)c == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(is.Take()); - c |= static_cast(static_cast(is.Take())) << 8; - c |= static_cast(static_cast(is.Take())) << 16; - c |= static_cast(static_cast(is.Take())) << 24; - return static_cast(c); + CharType c = (unsigned char)is.Take(); + c |= (unsigned char)is.Take() << 8; + c |= (unsigned char)is.Take() << 16; + c |= (unsigned char)is.Take() << 24; + return c; } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xFFu)); - os.Put(static_cast(0xFEu)); - os.Put(static_cast(0x00u)); - os.Put(static_cast(0x00u)); + os.Put(0xFFu); os.Put(0xFEu); os.Put(0x00u); os.Put(0x00u); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(c & 0xFFu)); - os.Put(static_cast((c >> 8) & 0xFFu)); - os.Put(static_cast((c >> 16) & 0xFFu)); - os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(c & 0xFFu); + os.Put((c >> 8) & 0xFFu); + os.Put((c >> 16) & 0xFFu); + os.Put((c >> 24) & 0xFFu); } }; @@ -498,35 +442,32 @@ struct UTF32BE : UTF32 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + return (unsigned)c == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(static_cast(is.Take())) << 24; - c |= static_cast(static_cast(is.Take())) << 16; - c |= static_cast(static_cast(is.Take())) << 8; - c |= static_cast(static_cast(is.Take())); - return static_cast(c); + CharType c = (unsigned char)is.Take() << 24; + c |= (unsigned char)is.Take() << 16; + c |= (unsigned char)is.Take() << 8; + c |= (unsigned char)is.Take(); + return c; } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0x00u)); - os.Put(static_cast(0x00u)); - os.Put(static_cast(0xFEu)); - os.Put(static_cast(0xFFu)); + os.Put(0x00u); os.Put(0x00u); os.Put(0xFEu); os.Put(0xFFu); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast((c >> 24) & 0xFFu)); - os.Put(static_cast((c >> 16) & 0xFFu)); - os.Put(static_cast((c >> 8) & 0xFFu)); - os.Put(static_cast(c & 0xFFu)); + os.Put((c >> 24) & 0xFFu); + os.Put((c >> 16) & 0xFFu); + os.Put((c >> 8) & 0xFFu); + os.Put(c & 0xFFu); } }; @@ -550,37 +491,31 @@ struct ASCII { os.Put(static_cast(codepoint & 0xFF)); } - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x7F); - PutUnsafe(os, static_cast(codepoint & 0xFF)); - } - template static bool Decode(InputStream& is, unsigned* codepoint) { - uint8_t c = static_cast(is.Take()); + unsigned char c = static_cast(is.Take()); *codepoint = c; return c <= 0X7F; } template static bool Validate(InputStream& is, OutputStream& os) { - uint8_t c = static_cast(is.Take()); - os.Put(static_cast(c)); + unsigned char c = is.Take(); + os.Put(c); return c <= 0x7F; } template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - uint8_t c = static_cast(Take(is)); - return static_cast(c); + Ch c = Take(is); + return c; } template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return static_cast(is.Take()); + return is.Take(); } template @@ -620,28 +555,21 @@ struct AutoUTF { #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x template - static RAPIDJSON_FORCEINLINE void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; (*f[os.GetType()])(os, codepoint); } - template - static RAPIDJSON_FORCEINLINE void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - typedef void (*EncodeFunc)(OutputStream&, unsigned); - static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; - (*f[os.GetType()])(os, codepoint); - } - template - static RAPIDJSON_FORCEINLINE bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { typedef bool (*DecodeFunc)(InputStream&, unsigned*); static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; return (*f[is.GetType()])(is, codepoint); } template - static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { typedef bool (*ValidateFunc)(InputStream&, OutputStream&); static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; return (*f[is.GetType()])(is, os); @@ -658,7 +586,7 @@ template struct Transcoder { //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. template - static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; @@ -666,50 +594,31 @@ struct Transcoder { return true; } - template - static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - TargetEncoding::EncodeUnsafe(os, codepoint); - return true; - } - //! Validate one Unicode codepoint from an encoded stream. template - static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { return Transcode(is, os); // Since source/target encoding is different, must transcode. } }; -// Forward declaration. -template -inline void PutUnsafe(Stream& stream, typename Stream::Ch c); - //! Specialization of Transcoder with same source and target encoding. template struct Transcoder { template - static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template - static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { - PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. - return true; - } - - template - static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { return Encoding::Validate(is, os); // source/target encoding are the same } }; RAPIDJSON_NAMESPACE_END -#if defined(__GNUC__) || defined(_MSC_VER) +#if defined(__GNUC__) || defined(_MSV_VER) RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/error/en.h b/contrib/rapidjson/include/rapidjson/error/en.h index 2db838bff..d5f9caab8 100644 --- a/contrib/rapidjson/include/rapidjson/error/en.h +++ b/contrib/rapidjson/include/rapidjson/error/en.h @@ -12,17 +12,11 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#ifndef RAPIDJSON_ERROR_EN_H_ -#define RAPIDJSON_ERROR_EN_H_ +#ifndef RAPIDJSON_ERROR_EN_H__ +#define RAPIDJSON_ERROR_EN_H__ #include "error.h" -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(covered-switch-default) -#endif - RAPIDJSON_NAMESPACE_BEGIN //! Maps error code of parsing into error message. @@ -38,7 +32,7 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); - case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); + case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); @@ -61,14 +55,11 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); - default: return RAPIDJSON_ERROR_STRING("Unknown error."); + default: + return RAPIDJSON_ERROR_STRING("Unknown error."); } } RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_ERROR_EN_H_ +#endif // RAPIDJSON_ERROR_EN_H__ diff --git a/contrib/rapidjson/include/rapidjson/error/error.h b/contrib/rapidjson/include/rapidjson/error/error.h index 9311d2f03..f9094fb95 100644 --- a/contrib/rapidjson/include/rapidjson/error/error.h +++ b/contrib/rapidjson/include/rapidjson/error/error.h @@ -12,16 +12,11 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#ifndef RAPIDJSON_ERROR_ERROR_H_ -#define RAPIDJSON_ERROR_ERROR_H_ +#ifndef RAPIDJSON_ERROR_ERROR_H__ +#define RAPIDJSON_ERROR_ERROR_H__ #include "../rapidjson.h" -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - /*! \file error.h */ /*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ @@ -104,9 +99,7 @@ enum ParseErrorCode { \see GenericReader::Parse, GenericDocument::Parse */ struct ParseResult { - //!! Unspecified boolean type - typedef bool (ParseResult::*BooleanType)() const; -public: + //! Default constructor, no error. ParseResult() : code_(kParseErrorNone), offset_(0) {} //! Constructor to set an error. @@ -117,8 +110,8 @@ public: //! Get the error offset, if \ref IsError(), 0 otherwise. size_t Offset() const { return offset_; } - //! Explicit conversion to \c bool, returns \c true, iff !\ref IsError(). - operator BooleanType() const { return !IsError() ? &ParseResult::IsError : NULL; } + //! Conversion to \c bool, returns \c true, iff !\ref IsError(). + operator bool() const { return !IsError(); } //! Whether the result is an error. bool IsError() const { return code_ != kParseErrorNone; } @@ -126,10 +119,6 @@ public: bool operator==(ParseErrorCode code) const { return code_ == code; } friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } - bool operator!=(const ParseResult& that) const { return !(*this == that); } - bool operator!=(ParseErrorCode code) const { return !(*this == code); } - friend bool operator!=(ParseErrorCode code, const ParseResult & err) { return err != code; } - //! Reset error code. void Clear() { Set(kParseErrorNone); } //! Update error code and offset. @@ -154,8 +143,4 @@ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_ERROR_ERROR_H_ +#endif // RAPIDJSON_ERROR_ERROR_H__ diff --git a/contrib/rapidjson/include/rapidjson/filereadstream.h b/contrib/rapidjson/include/rapidjson/filereadstream.h index b56ea13b3..3913eb74b 100644 --- a/contrib/rapidjson/include/rapidjson/filereadstream.h +++ b/contrib/rapidjson/include/rapidjson/filereadstream.h @@ -15,16 +15,9 @@ #ifndef RAPIDJSON_FILEREADSTREAM_H_ #define RAPIDJSON_FILEREADSTREAM_H_ -#include "stream.h" +#include "rapidjson.h" #include -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(unreachable-code) -RAPIDJSON_DIAG_OFF(missing-noreturn) -#endif - RAPIDJSON_NAMESPACE_BEGIN //! File byte stream for input using fread(). @@ -92,8 +85,4 @@ private: RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_FILESTREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/filewritestream.h b/contrib/rapidjson/include/rapidjson/filewritestream.h index 6378dd60e..dfb9cbd02 100644 --- a/contrib/rapidjson/include/rapidjson/filewritestream.h +++ b/contrib/rapidjson/include/rapidjson/filewritestream.h @@ -15,14 +15,9 @@ #ifndef RAPIDJSON_FILEWRITESTREAM_H_ #define RAPIDJSON_FILEWRITESTREAM_H_ -#include "stream.h" +#include "rapidjson.h" #include -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(unreachable-code) -#endif - RAPIDJSON_NAMESPACE_BEGIN //! Wrapper of C file stream for input using fread(). @@ -62,11 +57,7 @@ public: void Flush() { if (current_ != buffer_) { - size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); - if (result < static_cast(current_ - buffer_)) { - // failure deliberately ignored at this time - // added to avoid warn_unused_result build errors - } + fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); current_ = buffer_; } } @@ -97,8 +88,4 @@ inline void PutN(FileWriteStream& stream, char c, size_t n) { RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_FILESTREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/fwd.h b/contrib/rapidjson/include/rapidjson/fwd.h deleted file mode 100644 index e8104e841..000000000 --- a/contrib/rapidjson/include/rapidjson/fwd.h +++ /dev/null @@ -1,151 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_FWD_H_ -#define RAPIDJSON_FWD_H_ - -#include "rapidjson.h" - -RAPIDJSON_NAMESPACE_BEGIN - -// encodings.h - -template struct UTF8; -template struct UTF16; -template struct UTF16BE; -template struct UTF16LE; -template struct UTF32; -template struct UTF32BE; -template struct UTF32LE; -template struct ASCII; -template struct AutoUTF; - -template -struct Transcoder; - -// allocators.h - -class CrtAllocator; - -template -class MemoryPoolAllocator; - -// stream.h - -template -struct GenericStringStream; - -typedef GenericStringStream > StringStream; - -template -struct GenericInsituStringStream; - -typedef GenericInsituStringStream > InsituStringStream; - -// stringbuffer.h - -template -class GenericStringBuffer; - -typedef GenericStringBuffer, CrtAllocator> StringBuffer; - -// filereadstream.h - -class FileReadStream; - -// filewritestream.h - -class FileWriteStream; - -// memorybuffer.h - -template -struct GenericMemoryBuffer; - -typedef GenericMemoryBuffer MemoryBuffer; - -// memorystream.h - -struct MemoryStream; - -// reader.h - -template -struct BaseReaderHandler; - -template -class GenericReader; - -typedef GenericReader, UTF8, CrtAllocator> Reader; - -// writer.h - -template -class Writer; - -// prettywriter.h - -template -class PrettyWriter; - -// document.h - -template -struct GenericMember; - -template -class GenericMemberIterator; - -template -struct GenericStringRef; - -template -class GenericValue; - -typedef GenericValue, MemoryPoolAllocator > Value; - -template -class GenericDocument; - -typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; - -// pointer.h - -template -class GenericPointer; - -typedef GenericPointer Pointer; - -// schema.h - -template -class IGenericRemoteSchemaDocumentProvider; - -template -class GenericSchemaDocument; - -typedef GenericSchemaDocument SchemaDocument; -typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; - -template < - typename SchemaDocumentType, - typename OutputHandler, - typename StateAllocator> -class GenericSchemaValidator; - -typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/biginteger.h b/contrib/rapidjson/include/rapidjson/internal/biginteger.h index 9d3e88c99..99a30acf6 100644 --- a/contrib/rapidjson/include/rapidjson/internal/biginteger.h +++ b/contrib/rapidjson/include/rapidjson/internal/biginteger.h @@ -19,7 +19,6 @@ #if defined(_MSC_VER) && defined(_M_AMD64) #include // for _umul128 -#pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN @@ -51,16 +50,7 @@ public: if (length > 0) AppendDecimal64(decimals + i, decimals + i + length); } - - BigInteger& operator=(const BigInteger &rhs) - { - if (this != &rhs) { - count_ = rhs.count_; - std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); - } - return *this; - } - + BigInteger& operator=(uint64_t u) { digits_[0] = u; count_ = 1; @@ -240,7 +230,7 @@ private: uint64_t r = 0; for (const char* p = begin; p != end; ++p) { RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10u + static_cast(*p - '0'); + r = r * 10 + (*p - '0'); } return r; } diff --git a/contrib/rapidjson/include/rapidjson/internal/diyfp.h b/contrib/rapidjson/include/rapidjson/internal/diyfp.h index 29abf8046..3b6c4238c 100644 --- a/contrib/rapidjson/include/rapidjson/internal/diyfp.h +++ b/contrib/rapidjson/include/rapidjson/internal/diyfp.h @@ -21,10 +21,9 @@ #include "../rapidjson.h" -#if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) +#if defined(_MSC_VER) && defined(_M_AMD64) #include #pragma intrinsic(_BitScanReverse64) -#pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN @@ -35,13 +34,8 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - struct DiyFp { - DiyFp() : f(), e() {} + DiyFp() {} DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} @@ -238,8 +232,8 @@ inline DiyFp GetCachedPower(int e, int* K) { } inline DiyFp GetCachedPower10(int exp, int *outExp) { - unsigned index = (static_cast(exp) + 348u) / 8u; - *outExp = -348 + static_cast(index) * 8; + unsigned index = (exp + 348) / 8; + *outExp = -348 + index * 8; return GetCachedPowerByIndex(index); } @@ -247,11 +241,6 @@ inline DiyFp GetCachedPower10(int exp, int *outExp) { RAPIDJSON_DIAG_POP #endif -#ifdef __clang__ -RAPIDJSON_DIAG_POP -RAPIDJSON_DIAG_OFF(padded) -#endif - } // namespace internal RAPIDJSON_NAMESPACE_END diff --git a/contrib/rapidjson/include/rapidjson/internal/dtoa.h b/contrib/rapidjson/include/rapidjson/internal/dtoa.h index bf2e9b2e5..2d8d2e46a 100644 --- a/contrib/rapidjson/include/rapidjson/internal/dtoa.h +++ b/contrib/rapidjson/include/rapidjson/internal/dtoa.h @@ -29,7 +29,6 @@ namespace internal { #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) -RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 #endif inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { @@ -41,7 +40,7 @@ inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uin } } -inline int CountDecimalDigit32(uint32_t n) { +inline unsigned CountDecimalDigit32(uint32_t n) { // Simple pure C++ implementation was faster than __builtin_clz version in this situation. if (n < 10) return 1; if (n < 100) return 2; @@ -102,8 +101,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff kappa--; if (p2 < delta) { *K += kappa; - int index = -kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-kappa]); return; } } @@ -147,10 +145,10 @@ inline char* WriteExponent(int K, char* buffer) { return buffer; } -inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { +inline char* Prettify(char* buffer, int length, int k) { const int kk = length + k; // 10^(kk-1) <= v < 10^kk - if (0 <= k && kk <= 21) { + if (length <= kk && kk <= 21) { // 1234e7 -> 12340000000 for (int i = length; i < kk; i++) buffer[i] = '0'; @@ -160,44 +158,19 @@ inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { } else if (0 < kk && kk <= 21) { // 1234e-2 -> 12.34 - std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); + std::memmove(&buffer[kk + 1], &buffer[kk], length - kk); buffer[kk] = '.'; - if (0 > k + maxDecimalPlaces) { - // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 - // Remove extra trailing zeros (at least one) after truncation. - for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) - if (buffer[i] != '0') - return &buffer[i + 1]; - return &buffer[kk + 2]; // Reserve one zero - } - else - return &buffer[length + 1]; + return &buffer[length + 1]; } else if (-6 < kk && kk <= 0) { // 1234e-6 -> 0.001234 const int offset = 2 - kk; - std::memmove(&buffer[offset], &buffer[0], static_cast(length)); + std::memmove(&buffer[offset], &buffer[0], length); buffer[0] = '0'; buffer[1] = '.'; for (int i = 2; i < offset; i++) buffer[i] = '0'; - if (length - kk > maxDecimalPlaces) { - // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 - // Remove extra trailing zeros (at least one) after truncation. - for (int i = maxDecimalPlaces + 1; i > 2; i--) - if (buffer[i] != '0') - return &buffer[i + 1]; - return &buffer[3]; // Reserve one zero - } - else - return &buffer[length + offset]; - } - else if (kk < -maxDecimalPlaces) { - // Truncate to zero - buffer[0] = '0'; - buffer[1] = '.'; - buffer[2] = '0'; - return &buffer[3]; + return &buffer[length + offset]; } else if (length == 1) { // 1e30 @@ -206,15 +179,14 @@ inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { } else { // 1234e30 -> 1.234e33 - std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); + std::memmove(&buffer[2], &buffer[1], length - 1); buffer[1] = '.'; buffer[length + 1] = 'e'; return WriteExponent(kk - 1, &buffer[0 + length + 2]); } } -inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { - RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); +inline char* dtoa(double value, char* buffer) { Double d(value); if (d.IsZero()) { if (d.Sign()) @@ -231,7 +203,7 @@ inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { } int length, K; Grisu2(value, buffer, &length, &K); - return Prettify(buffer, length, K, maxDecimalPlaces); + return Prettify(buffer, length, K); } } diff --git a/contrib/rapidjson/include/rapidjson/internal/ieee754.h b/contrib/rapidjson/include/rapidjson/internal/ieee754.h index c2684ba2a..e3f03364c 100644 --- a/contrib/rapidjson/include/rapidjson/internal/ieee754.h +++ b/contrib/rapidjson/include/rapidjson/internal/ieee754.h @@ -40,7 +40,6 @@ public: bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } - bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } @@ -48,7 +47,7 @@ public: int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } - static int EffectiveSignificandSize(int order) { + static unsigned EffectiveSignificandSize(int order) { if (order >= -1021) return 53; else if (order <= -1074) diff --git a/contrib/rapidjson/include/rapidjson/internal/regex.h b/contrib/rapidjson/include/rapidjson/internal/regex.h deleted file mode 100644 index e1a2faae5..000000000 --- a/contrib/rapidjson/include/rapidjson/internal/regex.h +++ /dev/null @@ -1,734 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_REGEX_H_ -#define RAPIDJSON_INTERNAL_REGEX_H_ - -#include "../allocators.h" -#include "../stream.h" -#include "stack.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(implicit-fallthrough) -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#if __GNUC__ >= 7 -RAPIDJSON_DIAG_OFF(implicit-fallthrough) -#endif -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -#ifndef RAPIDJSON_REGEX_VERBOSE -#define RAPIDJSON_REGEX_VERBOSE 0 -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -/////////////////////////////////////////////////////////////////////////////// -// DecodedStream - -template -class DecodedStream { -public: - DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } - unsigned Peek() { return codepoint_; } - unsigned Take() { - unsigned c = codepoint_; - if (c) // No further decoding when '\0' - Decode(); - return c; - } - -private: - void Decode() { - if (!Encoding::Decode(ss_, &codepoint_)) - codepoint_ = 0; - } - - SourceStream& ss_; - unsigned codepoint_; -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericRegex - -static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 -static const SizeType kRegexInvalidRange = ~SizeType(0); - -template -class GenericRegexSearch; - -//! Regular expression engine with subset of ECMAscript grammar. -/*! - Supported regular expression syntax: - - \c ab Concatenation - - \c a|b Alternation - - \c a? Zero or one - - \c a* Zero or more - - \c a+ One or more - - \c a{3} Exactly 3 times - - \c a{3,} At least 3 times - - \c a{3,5} 3 to 5 times - - \c (ab) Grouping - - \c ^a At the beginning - - \c a$ At the end - - \c . Any character - - \c [abc] Character classes - - \c [a-c] Character class range - - \c [a-z0-9_] Character class combination - - \c [^abc] Negated character classes - - \c [^a-c] Negated character class range - - \c [\b] Backspace (U+0008) - - \c \\| \\\\ ... Escape characters - - \c \\f Form feed (U+000C) - - \c \\n Line feed (U+000A) - - \c \\r Carriage return (U+000D) - - \c \\t Tab (U+0009) - - \c \\v Vertical tab (U+000B) - - \note This is a Thompson NFA engine, implemented with reference to - Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", - https://swtch.com/~rsc/regexp/regexp1.html -*/ -template -class GenericRegex { -public: - typedef Encoding EncodingType; - typedef typename Encoding::Ch Ch; - template friend class GenericRegexSearch; - - GenericRegex(const Ch* source, Allocator* allocator = 0) : - states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), - anchorBegin_(), anchorEnd_() - { - GenericStringStream ss(source); - DecodedStream, Encoding> ds(ss); - Parse(ds); - } - - ~GenericRegex() {} - - bool IsValid() const { - return root_ != kRegexInvalidState; - } - -private: - enum Operator { - kZeroOrOne, - kZeroOrMore, - kOneOrMore, - kConcatenation, - kAlternation, - kLeftParenthesis - }; - - static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' - static const unsigned kRangeCharacterClass = 0xFFFFFFFE; - static const unsigned kRangeNegationFlag = 0x80000000; - - struct Range { - unsigned start; // - unsigned end; - SizeType next; - }; - - struct State { - SizeType out; //!< Equals to kInvalid for matching state - SizeType out1; //!< Equals to non-kInvalid for split - SizeType rangeStart; - unsigned codepoint; - }; - - struct Frag { - Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} - SizeType start; - SizeType out; //!< link-list of all output states - SizeType minIndex; - }; - - State& GetState(SizeType index) { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom()[index]; - } - - const State& GetState(SizeType index) const { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom()[index]; - } - - Range& GetRange(SizeType index) { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom()[index]; - } - - const Range& GetRange(SizeType index) const { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom()[index]; - } - - template - void Parse(DecodedStream& ds) { - Allocator allocator; - Stack operandStack(&allocator, 256); // Frag - Stack operatorStack(&allocator, 256); // Operator - Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) - - *atomCountStack.template Push() = 0; - - unsigned codepoint; - while (ds.Peek() != 0) { - switch (codepoint = ds.Take()) { - case '^': - anchorBegin_ = true; - break; - - case '$': - anchorEnd_ = true; - break; - - case '|': - while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - *operatorStack.template Push() = kAlternation; - *atomCountStack.template Top() = 0; - break; - - case '(': - *operatorStack.template Push() = kLeftParenthesis; - *atomCountStack.template Push() = 0; - break; - - case ')': - while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - if (operatorStack.Empty()) - return; - operatorStack.template Pop(1); - atomCountStack.template Pop(1); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '?': - if (!Eval(operandStack, kZeroOrOne)) - return; - break; - - case '*': - if (!Eval(operandStack, kZeroOrMore)) - return; - break; - - case '+': - if (!Eval(operandStack, kOneOrMore)) - return; - break; - - case '{': - { - unsigned n, m; - if (!ParseUnsigned(ds, &n)) - return; - - if (ds.Peek() == ',') { - ds.Take(); - if (ds.Peek() == '}') - m = kInfinityQuantifier; - else if (!ParseUnsigned(ds, &m) || m < n) - return; - } - else - m = n; - - if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') - return; - ds.Take(); - } - break; - - case '.': - PushOperand(operandStack, kAnyCharacterClass); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '[': - { - SizeType range; - if (!ParseRange(ds, &range)) - return; - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); - GetState(s).rangeStart = range; - *operandStack.template Push() = Frag(s, s, s); - } - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '\\': // Escape character - if (!CharacterEscape(ds, &codepoint)) - return; // Unsupported escape character - // fall through to default - - default: // Pattern character - PushOperand(operandStack, codepoint); - ImplicitConcatenation(atomCountStack, operatorStack); - } - } - - while (!operatorStack.Empty()) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - - // Link the operand to matching state. - if (operandStack.GetSize() == sizeof(Frag)) { - Frag* e = operandStack.template Pop(1); - Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); - root_ = e->start; - -#if RAPIDJSON_REGEX_VERBOSE - printf("root: %d\n", root_); - for (SizeType i = 0; i < stateCount_ ; i++) { - State& s = GetState(i); - printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); - } - printf("\n"); -#endif - } - } - - SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { - State* s = states_.template Push(); - s->out = out; - s->out1 = out1; - s->codepoint = codepoint; - s->rangeStart = kRegexInvalidRange; - return stateCount_++; - } - - void PushOperand(Stack& operandStack, unsigned codepoint) { - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); - *operandStack.template Push() = Frag(s, s, s); - } - - void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { - if (*atomCountStack.template Top()) - *operatorStack.template Push() = kConcatenation; - (*atomCountStack.template Top())++; - } - - SizeType Append(SizeType l1, SizeType l2) { - SizeType old = l1; - while (GetState(l1).out != kRegexInvalidState) - l1 = GetState(l1).out; - GetState(l1).out = l2; - return old; - } - - void Patch(SizeType l, SizeType s) { - for (SizeType next; l != kRegexInvalidState; l = next) { - next = GetState(l).out; - GetState(l).out = s; - } - } - - bool Eval(Stack& operandStack, Operator op) { - switch (op) { - case kConcatenation: - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); - { - Frag e2 = *operandStack.template Pop(1); - Frag e1 = *operandStack.template Pop(1); - Patch(e1.out, e2.start); - *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); - } - return true; - - case kAlternation: - if (operandStack.GetSize() >= sizeof(Frag) * 2) { - Frag e2 = *operandStack.template Pop(1); - Frag e1 = *operandStack.template Pop(1); - SizeType s = NewState(e1.start, e2.start, 0); - *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); - return true; - } - return false; - - case kZeroOrOne: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); - return true; - } - return false; - - case kZeroOrMore: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push() = Frag(s, s, e.minIndex); - return true; - } - return false; - - default: - RAPIDJSON_ASSERT(op == kOneOrMore); - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push() = Frag(e.start, s, e.minIndex); - return true; - } - return false; - } - } - - bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { - RAPIDJSON_ASSERT(n <= m); - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); - - if (n == 0) { - if (m == 0) // a{0} not support - return false; - else if (m == kInfinityQuantifier) - Eval(operandStack, kZeroOrMore); // a{0,} -> a* - else { - Eval(operandStack, kZeroOrOne); // a{0,5} -> a? - for (unsigned i = 0; i < m - 1; i++) - CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? - for (unsigned i = 0; i < m - 1; i++) - Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? - } - return true; - } - - for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a - CloneTopOperand(operandStack); - - if (m == kInfinityQuantifier) - Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ - else if (m > n) { - CloneTopOperand(operandStack); // a{3,5} -> a a a a - Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? - for (unsigned i = n; i < m - 1; i++) - CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? - for (unsigned i = n; i < m; i++) - Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? - } - - for (unsigned i = 0; i < n - 1; i++) - Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? - - return true; - } - - static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } - - void CloneTopOperand(Stack& operandStack) { - const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation - SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) - State* s = states_.template Push(count); - memcpy(s, &GetState(src.minIndex), count * sizeof(State)); - for (SizeType j = 0; j < count; j++) { - if (s[j].out != kRegexInvalidState) - s[j].out += count; - if (s[j].out1 != kRegexInvalidState) - s[j].out1 += count; - } - *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); - stateCount_ += count; - } - - template - bool ParseUnsigned(DecodedStream& ds, unsigned* u) { - unsigned r = 0; - if (ds.Peek() < '0' || ds.Peek() > '9') - return false; - while (ds.Peek() >= '0' && ds.Peek() <= '9') { - if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 - return false; // overflow - r = r * 10 + (ds.Take() - '0'); - } - *u = r; - return true; - } - - template - bool ParseRange(DecodedStream& ds, SizeType* range) { - bool isBegin = true; - bool negate = false; - int step = 0; - SizeType start = kRegexInvalidRange; - SizeType current = kRegexInvalidRange; - unsigned codepoint; - while ((codepoint = ds.Take()) != 0) { - if (isBegin) { - isBegin = false; - if (codepoint == '^') { - negate = true; - continue; - } - } - - switch (codepoint) { - case ']': - if (start == kRegexInvalidRange) - return false; // Error: nothing inside [] - if (step == 2) { // Add trailing '-' - SizeType r = NewRange('-'); - RAPIDJSON_ASSERT(current != kRegexInvalidRange); - GetRange(current).next = r; - } - if (negate) - GetRange(start).start |= kRangeNegationFlag; - *range = start; - return true; - - case '\\': - if (ds.Peek() == 'b') { - ds.Take(); - codepoint = 0x0008; // Escape backspace character - } - else if (!CharacterEscape(ds, &codepoint)) - return false; - // fall through to default - - default: - switch (step) { - case 1: - if (codepoint == '-') { - step++; - break; - } - // fall through to step 0 for other characters - - case 0: - { - SizeType r = NewRange(codepoint); - if (current != kRegexInvalidRange) - GetRange(current).next = r; - if (start == kRegexInvalidRange) - start = r; - current = r; - } - step = 1; - break; - - default: - RAPIDJSON_ASSERT(step == 2); - GetRange(current).end = codepoint; - step = 0; - } - } - } - return false; - } - - SizeType NewRange(unsigned codepoint) { - Range* r = ranges_.template Push(); - r->start = r->end = codepoint; - r->next = kRegexInvalidRange; - return rangeCount_++; - } - - template - bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { - unsigned codepoint; - switch (codepoint = ds.Take()) { - case '^': - case '$': - case '|': - case '(': - case ')': - case '?': - case '*': - case '+': - case '.': - case '[': - case ']': - case '{': - case '}': - case '\\': - *escapedCodepoint = codepoint; return true; - case 'f': *escapedCodepoint = 0x000C; return true; - case 'n': *escapedCodepoint = 0x000A; return true; - case 'r': *escapedCodepoint = 0x000D; return true; - case 't': *escapedCodepoint = 0x0009; return true; - case 'v': *escapedCodepoint = 0x000B; return true; - default: - return false; // Unsupported escape character - } - } - - Stack states_; - Stack ranges_; - SizeType root_; - SizeType stateCount_; - SizeType rangeCount_; - - static const unsigned kInfinityQuantifier = ~0u; - - // For SearchWithAnchoring() - bool anchorBegin_; - bool anchorEnd_; -}; - -template -class GenericRegexSearch { -public: - typedef typename RegexType::EncodingType Encoding; - typedef typename Encoding::Ch Ch; - - GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : - regex_(regex), allocator_(allocator), ownAllocator_(0), - state0_(allocator, 0), state1_(allocator, 0), stateSet_() - { - RAPIDJSON_ASSERT(regex_.IsValid()); - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); - state0_.template Reserve(regex_.stateCount_); - state1_.template Reserve(regex_.stateCount_); - } - - ~GenericRegexSearch() { - Allocator::Free(stateSet_); - RAPIDJSON_DELETE(ownAllocator_); - } - - template - bool Match(InputStream& is) { - return SearchWithAnchoring(is, true, true); - } - - bool Match(const Ch* s) { - GenericStringStream is(s); - return Match(is); - } - - template - bool Search(InputStream& is) { - return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); - } - - bool Search(const Ch* s) { - GenericStringStream is(s); - return Search(is); - } - -private: - typedef typename RegexType::State State; - typedef typename RegexType::Range Range; - - template - bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { - DecodedStream ds(is); - - state0_.Clear(); - Stack *current = &state0_, *next = &state1_; - const size_t stateSetSize = GetStateSetSize(); - std::memset(stateSet_, 0, stateSetSize); - - bool matched = AddState(*current, regex_.root_); - unsigned codepoint; - while (!current->Empty() && (codepoint = ds.Take()) != 0) { - std::memset(stateSet_, 0, stateSetSize); - next->Clear(); - matched = false; - for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { - const State& sr = regex_.GetState(*s); - if (sr.codepoint == codepoint || - sr.codepoint == RegexType::kAnyCharacterClass || - (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) - { - matched = AddState(*next, sr.out) || matched; - if (!anchorEnd && matched) - return true; - } - if (!anchorBegin) - AddState(*next, regex_.root_); - } - internal::Swap(current, next); - } - - return matched; - } - - size_t GetStateSetSize() const { - return (regex_.stateCount_ + 31) / 32 * 4; - } - - // Return whether the added states is a match state - bool AddState(Stack& l, SizeType index) { - RAPIDJSON_ASSERT(index != kRegexInvalidState); - - const State& s = regex_.GetState(index); - if (s.out1 != kRegexInvalidState) { // Split - bool matched = AddState(l, s.out); - return AddState(l, s.out1) || matched; - } - else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { - stateSet_[index >> 5] |= (1u << (index & 31)); - *l.template PushUnsafe() = index; - } - return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. - } - - bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { - bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; - while (rangeIndex != kRegexInvalidRange) { - const Range& r = regex_.GetRange(rangeIndex); - if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) - return yes; - rangeIndex = r.next; - } - return !yes; - } - - const RegexType& regex_; - Allocator* allocator_; - Allocator* ownAllocator_; - Stack state0_; - Stack state1_; - uint32_t* stateSet_; -}; - -typedef GenericRegex > Regex; -typedef GenericRegexSearch RegexSearch; - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/stack.h b/contrib/rapidjson/include/rapidjson/internal/stack.h index 5c5398c35..722d56923 100644 --- a/contrib/rapidjson/include/rapidjson/internal/stack.h +++ b/contrib/rapidjson/include/rapidjson/internal/stack.h @@ -15,13 +15,7 @@ #ifndef RAPIDJSON_INTERNAL_STACK_H_ #define RAPIDJSON_INTERNAL_STACK_H_ -#include "../allocators.h" -#include "swap.h" - -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif +#include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -38,6 +32,7 @@ public: // Optimization note: Do not allocate memory for stack_ in constructor. // Do it lazily when first Push() -> Expand() -> Resize(). Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { + RAPIDJSON_ASSERT(stackCapacity > 0); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -86,15 +81,6 @@ public: } #endif - void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { - internal::Swap(allocator_, rhs.allocator_); - internal::Swap(ownAllocator_, rhs.ownAllocator_); - internal::Swap(stack_, rhs.stack_); - internal::Swap(stackTop_, rhs.stackTop_); - internal::Swap(stackEnd_, rhs.stackEnd_); - internal::Swap(initialCapacity_, rhs.initialCapacity_); - } - void Clear() { stackTop_ = stack_; } void ShrinkToFit() { @@ -111,23 +97,12 @@ public: // Optimization note: try to minimize the size of this function for force inline. // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. - template - RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { - // Expand the stack if needed - if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) - Expand(count); - } - template RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { - Reserve(count); - return PushUnsafe(count); - } + // Expand the stack if needed + if (stackTop_ + sizeof(T) * count >= stackEnd_) + Expand(count); - template - RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { - RAPIDJSON_ASSERT(stackTop_); - RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; @@ -147,32 +122,9 @@ public: } template - const T* Top() const { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return reinterpret_cast(stackTop_ - sizeof(T)); - } - - template - T* End() { return reinterpret_cast(stackTop_); } - - template - const T* End() const { return reinterpret_cast(stackTop_); } - - template - T* Bottom() { return reinterpret_cast(stack_); } - - template - const T* Bottom() const { return reinterpret_cast(stack_); } - - bool HasAllocator() const { - return allocator_ != 0; - } - - Allocator& GetAllocator() { - RAPIDJSON_ASSERT(allocator_); - return *allocator_; - } + T* Bottom() { return (T*)stack_; } + Allocator& GetAllocator() { return *allocator_; } bool Empty() const { return stackTop_ == stack_; } size_t GetSize() const { return static_cast(stackTop_ - stack_); } size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } @@ -184,7 +136,7 @@ private: size_t newCapacity; if (stack_ == 0) { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); newCapacity = initialCapacity_; } else { newCapacity = GetCapacity(); @@ -199,7 +151,7 @@ private: void Resize(size_t newCapacity) { const size_t size = GetSize(); // Backup the current size - stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); + stack_ = (char*)allocator_->Realloc(stack_, GetCapacity(), newCapacity); stackTop_ = stack_ + size; stackEnd_ = stack_ + newCapacity; } @@ -224,8 +176,4 @@ private: } // namespace internal RAPIDJSON_NAMESPACE_END -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_STACK_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/strfunc.h b/contrib/rapidjson/include/rapidjson/internal/strfunc.h index 226439a76..84405065a 100644 --- a/contrib/rapidjson/include/rapidjson/internal/strfunc.h +++ b/contrib/rapidjson/include/rapidjson/internal/strfunc.h @@ -15,8 +15,7 @@ #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ #define RAPIDJSON_INTERNAL_STRFUNC_H_ -#include "../stream.h" -#include +#include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -29,40 +28,11 @@ namespace internal { */ template inline SizeType StrLen(const Ch* s) { - RAPIDJSON_ASSERT(s != 0); const Ch* p = s; while (*p) ++p; return SizeType(p - s); } -template <> -inline SizeType StrLen(const char* s) { - return SizeType(std::strlen(s)); -} - -template <> -inline SizeType StrLen(const wchar_t* s) { - return SizeType(std::wcslen(s)); -} - -//! Returns number of code points in a encoded string. -template -bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { - RAPIDJSON_ASSERT(s != 0); - RAPIDJSON_ASSERT(outCount != 0); - GenericStringStream is(s); - const typename Encoding::Ch* end = s + length; - SizeType count = 0; - while (is.src_ < end) { - unsigned codepoint; - if (!Encoding::Decode(is, &codepoint)) - return false; - count++; - } - *outCount = count; - return true; -} - } // namespace internal RAPIDJSON_NAMESPACE_END diff --git a/contrib/rapidjson/include/rapidjson/internal/strtod.h b/contrib/rapidjson/include/rapidjson/internal/strtod.h index adf49e349..ace65f677 100644 --- a/contrib/rapidjson/include/rapidjson/internal/strtod.h +++ b/contrib/rapidjson/include/rapidjson/internal/strtod.h @@ -15,6 +15,7 @@ #ifndef RAPIDJSON_STRTOD_ #define RAPIDJSON_STRTOD_ +#include "../rapidjson.h" #include "ieee754.h" #include "biginteger.h" #include "diyfp.h" @@ -94,13 +95,13 @@ inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { hS_Exp2 -= common_Exp2; BigInteger dS = d; - dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); + dS.MultiplyPow5(dS_Exp5) <<= dS_Exp2; BigInteger bS(bInt); - bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); + bS.MultiplyPow5(bS_Exp5) <<= bS_Exp2; BigInteger hS(1); - hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); + hS.MultiplyPow5(hS_Exp5) <<= hS_Exp2; BigInteger delta(0); dS.Difference(bS, &delta); @@ -133,22 +134,22 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) break; - significand = significand * 10u + static_cast(decimals[i] - '0'); + significand = significand * 10 + (decimals[i] - '0'); } if (i < length && decimals[i] >= '5') // Rounding significand++; size_t remaining = length - i; - const int kUlpShift = 3; - const int kUlp = 1 << kUlpShift; - int64_t error = (remaining == 0) ? 0 : kUlp / 2; + const unsigned kUlpShift = 3; + const unsigned kUlp = 1 << kUlpShift; + int error = (remaining == 0) ? 0 : kUlp / 2; DiyFp v(significand, 0); v = v.Normalize(); error <<= -v.e; - const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; + const int dExp = (int)decimalPosition - (int)i + exp; int actualExp; DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); @@ -162,10 +163,10 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 }; - int adjustment = dExp - actualExp - 1; + int adjustment = dExp - actualExp - 1; RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); v = v * kPow10[adjustment]; - if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit + if (length + adjustment > 19) // has more digits than decimal digits in 64-bit error += kUlp / 2; } @@ -177,10 +178,10 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit v = v.Normalize(); error <<= oldExp - v.e; - const int effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); - int precisionSize = 64 - effectiveSignificandSize; + const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + unsigned precisionSize = 64 - effectiveSignificandSize; if (precisionSize + kUlpShift >= 64) { - int scaleExp = (precisionSize + kUlpShift) - 63; + unsigned scaleExp = (precisionSize + kUlpShift) - 63; v.f >>= scaleExp; v.e += scaleExp; error = (error >> scaleExp) + 1 + kUlp; @@ -190,7 +191,7 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit DiyFp rounded(v.f >> precisionSize, v.e + precisionSize); const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; - if (precisionBits >= halfWay + static_cast(error)) { + if (precisionBits >= halfWay + error) { rounded.f++; if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) rounded.f >>= 1; @@ -200,12 +201,12 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit *result = rounded.ToDouble(); - return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); + return halfWay - error >= precisionBits || precisionBits >= halfWay + error; } inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { const BigInteger dInt(decimals, length); - const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; + const int dExp = (int)decimalPosition - (int)length + exp; Double a(approx); int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); if (cmp < 0) @@ -245,10 +246,10 @@ inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t // Trim right-most digits const int kMaxDecimalDigit = 780; - if (static_cast(length) > kMaxDecimalDigit) { - int delta = (static_cast(length) - kMaxDecimalDigit); + if ((int)length > kMaxDecimalDigit) { + int delta = (int(length) - kMaxDecimalDigit); exp += delta; - decimalPosition -= static_cast(delta); + decimalPosition -= delta; length = kMaxDecimalDigit; } diff --git a/contrib/rapidjson/include/rapidjson/internal/swap.h b/contrib/rapidjson/include/rapidjson/internal/swap.h index 666e49f97..0590921f1 100644 --- a/contrib/rapidjson/include/rapidjson/internal/swap.h +++ b/contrib/rapidjson/include/rapidjson/internal/swap.h @@ -17,11 +17,6 @@ #include "../rapidjson.h" -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -39,8 +34,4 @@ inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { } // namespace internal RAPIDJSON_NAMESPACE_END -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/contrib/rapidjson/include/rapidjson/istreamwrapper.h b/contrib/rapidjson/include/rapidjson/istreamwrapper.h deleted file mode 100644 index 8639c8c3c..000000000 --- a/contrib/rapidjson/include/rapidjson/istreamwrapper.h +++ /dev/null @@ -1,115 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ISTREAMWRAPPER_H_ -#define RAPIDJSON_ISTREAMWRAPPER_H_ - -#include "stream.h" -#include - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. -/*! - The classes can be wrapped including but not limited to: - - - \c std::istringstream - - \c std::stringstream - - \c std::wistringstream - - \c std::wstringstream - - \c std::ifstream - - \c std::fstream - - \c std::wifstream - - \c std::wfstream - - \tparam StreamType Class derived from \c std::basic_istream. -*/ - -template -class BasicIStreamWrapper { -public: - typedef typename StreamType::char_type Ch; - BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} - - Ch Peek() const { - typename StreamType::int_type c = stream_.peek(); - return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : static_cast('\0'); - } - - Ch Take() { - typename StreamType::int_type c = stream_.get(); - if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { - count_++; - return static_cast(c); - } - else - return '\0'; - } - - // tellg() may return -1 when failed. So we count by ourself. - size_t Tell() const { return count_; } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. - int i; - bool hasError = false; - for (i = 0; i < 4; ++i) { - typename StreamType::int_type c = stream_.get(); - if (c == StreamType::traits_type::eof()) { - hasError = true; - stream_.clear(); - break; - } - peekBuffer_[i] = static_cast(c); - } - for (--i; i >= 0; --i) - stream_.putback(peekBuffer_[i]); - return !hasError ? peekBuffer_ : 0; - } - -private: - BasicIStreamWrapper(const BasicIStreamWrapper&); - BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); - - StreamType& stream_; - size_t count_; //!< Number of characters read. Note: - mutable Ch peekBuffer_[4]; -}; - -typedef BasicIStreamWrapper IStreamWrapper; -typedef BasicIStreamWrapper WIStreamWrapper; - -#if defined(__clang__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/contrib/rapidjson/include/rapidjson/memorybuffer.h b/contrib/rapidjson/include/rapidjson/memorybuffer.h index 39bee1dec..2484b2185 100644 --- a/contrib/rapidjson/include/rapidjson/memorybuffer.h +++ b/contrib/rapidjson/include/rapidjson/memorybuffer.h @@ -15,7 +15,7 @@ #ifndef RAPIDJSON_MEMORYBUFFER_H_ #define RAPIDJSON_MEMORYBUFFER_H_ -#include "stream.h" +#include "rapidjson.h" #include "internal/stack.h" RAPIDJSON_NAMESPACE_BEGIN diff --git a/contrib/rapidjson/include/rapidjson/memorystream.h b/contrib/rapidjson/include/rapidjson/memorystream.h index 1d71d8a4f..99feae5d7 100644 --- a/contrib/rapidjson/include/rapidjson/memorystream.h +++ b/contrib/rapidjson/include/rapidjson/memorystream.h @@ -15,13 +15,7 @@ #ifndef RAPIDJSON_MEMORYSTREAM_H_ #define RAPIDJSON_MEMORYSTREAM_H_ -#include "stream.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(unreachable-code) -RAPIDJSON_DIAG_OFF(missing-noreturn) -#endif +#include "rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN @@ -42,8 +36,8 @@ struct MemoryStream { MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} - Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } - Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } + Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } + Ch Take() { return (src_ == end_) ? '\0' : *src_++; } size_t Tell() const { return static_cast(src_ - begin_); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } @@ -64,8 +58,4 @@ struct MemoryStream { RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h b/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h index 3d4477b9a..a26fff4bf 100644 --- a/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h +++ b/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h @@ -89,14 +89,14 @@ #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we have to wrap include with 'extern "C++" {}' -// or compiler would give many errors like this: +// compiling for ARM we should wrap include with 'extern "C++" {}' +// or compiler give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#if defined(__cplusplus) && !defined(_M_ARM) +#ifdef __cplusplus extern "C" { #endif # include -#if defined(__cplusplus) && !defined(_M_ARM) +#ifdef __cplusplus } #endif diff --git a/contrib/rapidjson/include/rapidjson/ostreamwrapper.h b/contrib/rapidjson/include/rapidjson/ostreamwrapper.h deleted file mode 100644 index 6f4667c08..000000000 --- a/contrib/rapidjson/include/rapidjson/ostreamwrapper.h +++ /dev/null @@ -1,81 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ -#define RAPIDJSON_OSTREAMWRAPPER_H_ - -#include "stream.h" -#include - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. -/*! - The classes can be wrapped including but not limited to: - - - \c std::ostringstream - - \c std::stringstream - - \c std::wpstringstream - - \c std::wstringstream - - \c std::ifstream - - \c std::fstream - - \c std::wofstream - - \c std::wfstream - - \tparam StreamType Class derived from \c std::basic_ostream. -*/ - -template -class BasicOStreamWrapper { -public: - typedef typename StreamType::char_type Ch; - BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} - - void Put(Ch c) { - stream_.put(c); - } - - void Flush() { - stream_.flush(); - } - - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - BasicOStreamWrapper(const BasicOStreamWrapper&); - BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); - - StreamType& stream_; -}; - -typedef BasicOStreamWrapper OStreamWrapper; -typedef BasicOStreamWrapper WOStreamWrapper; - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/contrib/rapidjson/include/rapidjson/pointer.h b/contrib/rapidjson/include/rapidjson/pointer.h index 0f377efec..5d2aa8d63 100644 --- a/contrib/rapidjson/include/rapidjson/pointer.h +++ b/contrib/rapidjson/include/rapidjson/pointer.h @@ -18,16 +18,6 @@ #include "document.h" #include "internal/itoa.h" -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(switch-enum) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - RAPIDJSON_NAMESPACE_BEGIN static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token @@ -81,7 +71,7 @@ template class GenericPointer { public: typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value - typedef typename ValueType::Ch Ch; //!< Character type from Value + typedef typename EncodingType::Ch Ch; //!< Character type from Value //! A token is the basic units of internal representation. /*! @@ -106,7 +96,7 @@ public: //@{ //! Default constructor. - GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + GenericPointer() : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Constructor that parses a string or URI fragment representation. /*! @@ -165,7 +155,7 @@ public: GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Copy constructor. - GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + GenericPointer(const GenericPointer& rhs) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { *this = rhs; } @@ -240,7 +230,7 @@ public: template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) Append(T* name, Allocator* allocator = 0) const { - return Append(name, internal::StrLen(name), allocator); + return Append(name, StrLen(name), allocator); } #if RAPIDJSON_HAS_STDSTRING @@ -263,18 +253,17 @@ public: */ GenericPointer Append(SizeType index, Allocator* allocator = 0) const { char buffer[21]; - char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); - SizeType length = static_cast(end - buffer); + SizeType length = (sizeof(SizeType) == 4 ? internal::u32toa(index, buffer): internal::u64toa(index, buffer)) - buffer; buffer[length] = '\0'; if (sizeof(Ch) == 1) { - Token token = { reinterpret_cast(buffer), length, index }; + Token token = { (Ch*)buffer, length, index }; return Append(token, allocator); } else { Ch name[21]; for (size_t i = 0; i <= length; i++) - name[i] = static_cast(buffer[i]); + name[i] = buffer[i]; Token token = { name, length, index }; return Append(token, allocator); } @@ -282,7 +271,7 @@ public: //! Append a token by value, and return a new Pointer /*! - \param token token to be appended. + \param value Value (either Uint or String) to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ @@ -310,9 +299,6 @@ public: //@} - //! Get the allocator of this pointer. - Allocator& GetAllocator() { return *allocator_; } - //!@name Tokens //@{ @@ -404,7 +390,7 @@ public: bool exist = true; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { if (v->IsArray() && t->name[0] == '-' && t->length == 1) { - v->PushBack(ValueType().Move(), allocator); + v->PushBack(Value().Move(), allocator); v = &((*v)[v->Size() - 1]); exist = false; } @@ -422,7 +408,7 @@ public: if (t->index >= v->Size()) { v->Reserve(t->index + 1, allocator); while (t->index >= v->Size()) - v->PushBack(ValueType().Move(), allocator); + v->PushBack(Value().Move(), allocator); exist = false; } v = &((*v)[t->index]); @@ -430,7 +416,7 @@ public: else { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) { - v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); + v->AddMember(Value(t->name, t->length, allocator).Move(), Value().Move(), allocator); v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end exist = false; } @@ -449,6 +435,7 @@ public: //! Creates a value in a document. /*! \param document A document to be resolved. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. \param alreadyExist If non-null, it stores whether the resolved value is already exist. \return The resolved newly created, or already exists value. */ @@ -465,18 +452,9 @@ public: //! Query a value in a subtree. /*! \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. \return Pointer to the value if it can be resolved. Otherwise null. - - \note - There are only 3 situations when a value cannot be resolved: - 1. A value in the path is not an array nor object. - 2. An object value does not contain the token. - 3. A token is out of range of an array value. - - Use unresolvedTokenIndex to retrieve the token index. */ - ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { + ValueType* Get(ValueType& root) const { RAPIDJSON_ASSERT(IsValid()); ValueType* v = &root; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { @@ -485,23 +463,18 @@ public: { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) - break; + return 0; v = &m->value; } - continue; + break; case kArrayType: if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - break; + return 0; v = &((*v)[t->index]); - continue; - default: break; + default: + return 0; } - - // Error: unresolved token - if (unresolvedTokenIndex) - *unresolvedTokenIndex = static_cast(t - tokens_); - return 0; } return v; } @@ -511,9 +484,7 @@ public: \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \return Pointer to the value if it can be resolved. Otherwise null. */ - const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { - return Get(const_cast(root), unresolvedTokenIndex); - } + const ValueType* Get(const ValueType& root) const { return Get(const_cast(root)); } //@} @@ -554,7 +525,7 @@ public: //! Query a value in a subtree with default primitive value. /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -584,7 +555,7 @@ public: //! Query a value in a document with default primitive value. /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -630,7 +601,7 @@ public: //! Set a primitive value in a subtree. /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -666,7 +637,7 @@ public: //! Set a primitive value in a document. /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -758,7 +729,7 @@ private: */ Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) @@ -767,12 +738,8 @@ private: tokenCount_ = rhs.tokenCount_ + extraToken; tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - if (rhs.tokenCount_ > 0) { - std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); - } - if (nameBufferSize > 0) { - std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); - } + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); // Adjust pointers to name buffer std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; @@ -792,13 +759,11 @@ private: } //! Parse a JSON String or its URI fragment representation into tokens. -#ifndef __clang__ // -Wdocumentation /*! \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. \param length Length of the source string. \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. */ -#endif void Parse(const Ch* source, size_t length) { RAPIDJSON_ASSERT(source != NULL); RAPIDJSON_ASSERT(nameBuffer_ == 0); @@ -806,7 +771,7 @@ private: // Create own allocator if user did not supply. if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); // Count number of '/' as tokenCount tokenCount_ = 0; @@ -892,7 +857,7 @@ private: *name++ = c; } - token->length = static_cast(name - token->name); + token->length = name - token->name; if (token->length == 0) isNumber = false; *name++ = '\0'; // Null terminator @@ -979,8 +944,6 @@ private: */ class PercentDecodeStream { public: - typedef typename ValueType::Ch Ch; - //! Constructor /*! \param source Start of the stream @@ -996,11 +959,11 @@ private: src_++; Ch c = 0; for (int j = 0; j < 2; j++) { - c = static_cast(c << 4); + c <<= 4; Ch h = *src_; - if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); - else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); - else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); + if (h >= '0' && h <= '9') c += h - '0'; + else if (h >= 'A' && h <= 'F') c += h - 'A' + 10; + else if (h >= 'a' && h <= 'f') c += h - 'a' + 10; else { valid_ = false; return 0; @@ -1010,7 +973,7 @@ private: return c; } - size_t Tell() const { return static_cast(src_ - head_); } + size_t Tell() const { return src_ - head_; } bool IsValid() const { return valid_; } private: @@ -1029,8 +992,8 @@ private: unsigned char u = static_cast(c); static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; os_.Put('%'); - os_.Put(static_cast(hexDigits[u >> 4])); - os_.Put(static_cast(hexDigits[u & 15])); + os_.Put(hexDigits[u >> 4]); + os_.Put(hexDigits[u & 15]); } private: OutputStream& os_; @@ -1078,23 +1041,23 @@ typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, c ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer) { + return pointer.Get(root); } template -const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer) { + return pointer.Get(root); } template -typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N]) { + return GenericPointer(source, N - 1).Get(root); } template -const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Get(root); } ////////////////////////////////////////////////////////////////////////////// @@ -1347,12 +1310,4 @@ bool EraseValueByPointer(T& root, const CharType(&source)[N]) { RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_POINTER_H_ diff --git a/contrib/rapidjson/include/rapidjson/prettywriter.h b/contrib/rapidjson/include/rapidjson/prettywriter.h index 98dfb3060..416dd492e 100644 --- a/contrib/rapidjson/include/rapidjson/prettywriter.h +++ b/contrib/rapidjson/include/rapidjson/prettywriter.h @@ -22,21 +22,8 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - RAPIDJSON_NAMESPACE_BEGIN -//! Combination of PrettyWriter format flags. -/*! \see PrettyWriter::SetFormatOptions - */ -enum PrettyFormatOptions { - kFormatDefault = 0, //!< Default pretty formatting. - kFormatSingleLineArray = 1 //!< Format arrays on a single line. -}; - //! Writer with indentation and spacing. /*! \tparam OutputStream Type of ouptut os. @@ -44,10 +31,10 @@ enum PrettyFormatOptions { \tparam TargetEncoding Encoding of output stream. \tparam StackAllocator Type of allocator for allocating memory of stack. */ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> -class PrettyWriter : public Writer { +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> +class PrettyWriter : public Writer { public: - typedef Writer Base; + typedef Writer Base; typedef typename Base::Ch Ch; //! Constructor @@ -55,17 +42,8 @@ public: \param allocator User supplied allocator. If it is null, it will create a private one. \param levelDepth Initial capacity of stack. */ - explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} - - - explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - PrettyWriter(PrettyWriter&& rhs) : - Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} -#endif + PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} //! Set custom indentation. /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). @@ -79,14 +57,6 @@ public: return *this; } - //! Set pretty writer formatting options. - /*! \param options Formatting options. - */ - PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { - formatOptions_ = options; - return *this; - } - /*! @name Implementation of Handler \see Handler */ @@ -100,15 +70,7 @@ public: bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - RAPIDJSON_ASSERT(str != 0); - (void)copy; - PrettyPrefix(kNumberType); - return Base::WriteString(str, length); - } - bool String(const Ch* str, SizeType length, bool copy = false) { - RAPIDJSON_ASSERT(str != 0); (void)copy; PrettyPrefix(kStringType); return Base::WriteString(str, length); @@ -127,19 +89,11 @@ public: } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - -#if RAPIDJSON_HAS_STDSTRING - bool Key(const std::basic_string& str) { - return Key(str.data(), SizeType(str.size())); - } -#endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object - RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value - + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; if (!empty) { @@ -150,7 +104,7 @@ public: (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::Flush(); + Base::os_->Flush(); return true; } @@ -166,7 +120,7 @@ public: RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { + if (!empty) { Base::os_->Put('\n'); WriteIndent(); } @@ -174,7 +128,7 @@ public: (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::Flush(); + Base::os_->Flush(); return true; } @@ -188,22 +142,6 @@ public: bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } //@} - - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. - - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. - */ - bool RawValue(const Ch* json, size_t length, Type type) { - RAPIDJSON_ASSERT(json != 0); - PrettyPrefix(type); - return Base::WriteRawValue(json, length); - } - protected: void PrettyPrefix(Type type) { (void)type; @@ -213,14 +151,11 @@ protected: if (level->inArray) { if (level->valueCount > 0) { Base::os_->Put(','); // add comma if it is not the first element in array - if (formatOptions_ & kFormatSingleLineArray) - Base::os_->Put(' '); - } - - if (!(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); - WriteIndent(); } + else + Base::os_->Put('\n'); + WriteIndent(); } else { // in object if (level->valueCount > 0) { @@ -251,12 +186,11 @@ protected: void WriteIndent() { size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, static_cast(indentChar_), count); + PutN(*Base::os_, indentChar_, count); } Ch indentChar_; unsigned indentCharCount_; - PrettyFormatOptions formatOptions_; private: // Prohibit copy constructor & assignment operator. @@ -266,10 +200,6 @@ private: RAPIDJSON_NAMESPACE_END -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/rapidjson.h b/contrib/rapidjson/include/rapidjson/rapidjson.h index 5716fdc06..f22130d3c 100644 --- a/contrib/rapidjson/include/rapidjson/rapidjson.h +++ b/contrib/rapidjson/include/rapidjson/rapidjson.h @@ -49,11 +49,6 @@ // token stringification #define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) #define RAPIDJSON_DO_STRINGIFY(x) #x - -// token concatenation -#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) -#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) -#define RAPIDJSON_DO_JOIN2(X, Y) X##Y //!@endcond /*! \def RAPIDJSON_MAJOR_VERSION @@ -73,8 +68,8 @@ \brief Version of RapidJSON in ".." string format. */ #define RAPIDJSON_MAJOR_VERSION 1 -#define RAPIDJSON_MINOR_VERSION 1 -#define RAPIDJSON_PATCH_VERSION 0 +#define RAPIDJSON_MINOR_VERSION 0 +#define RAPIDJSON_PATCH_VERSION 2 #define RAPIDJSON_VERSION_STRING \ RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) @@ -124,31 +119,6 @@ #define RAPIDJSON_NAMESPACE_END } #endif -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_HAS_STDSTRING - -#ifndef RAPIDJSON_HAS_STDSTRING -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation -#else -#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default -#endif -/*! \def RAPIDJSON_HAS_STDSTRING - \ingroup RAPIDJSON_CONFIG - \brief Enable RapidJSON support for \c std::string - - By defining this preprocessor symbol to \c 1, several convenience functions for using - \ref rapidjson::GenericValue with \c std::string are enabled, especially - for construction and comparison. - - \hideinitializer -*/ -#endif // !defined(RAPIDJSON_HAS_STDSTRING) - -#if RAPIDJSON_HAS_STDSTRING -#include -#endif // RAPIDJSON_HAS_STDSTRING - /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE @@ -164,7 +134,7 @@ */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#ifdef _MSC_VER #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else @@ -183,9 +153,9 @@ #ifndef RAPIDJSON_FORCEINLINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && defined(NDEBUG) +#if defined(_MSC_VER) && !defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __forceinline -#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) +#elif defined(__GNUC__) && __GNUC__ >= 4 && !defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) #else #define RAPIDJSON_FORCEINLINE @@ -241,8 +211,6 @@ # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else @@ -255,7 +223,7 @@ //! Whether using 64-bit architecture #ifndef RAPIDJSON_64BIT -#if defined(__LP64__) || (defined(__x86_64__) && defined(__ILP32__)) || defined(_WIN64) || defined(__EMSCRIPTEN__) +#if defined(__LP64__) || defined(_WIN64) #define RAPIDJSON_64BIT 1 #else #define RAPIDJSON_64BIT 0 @@ -270,14 +238,13 @@ \param x pointer to align Some machines require strict data alignment. Currently the default uses 4 bytes - alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. - User can customize by defining the RAPIDJSON_ALIGN function macro. + alignment. User can customize by defining the RAPIDJSON_ALIGN function macro., */ #ifndef RAPIDJSON_ALIGN #if RAPIDJSON_64BIT == 1 -#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) +#define RAPIDJSON_ALIGN(x) ((x + 7u) & ~7u) #else -#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) +#define RAPIDJSON_ALIGN(x) ((x + 3u) & ~3u) #endif #endif @@ -295,47 +262,17 @@ #endif /////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_48BITPOINTER_OPTIMIZATION - -//! Use only lower 48-bit address for some pointers. -/*! - \ingroup RAPIDJSON_CONFIG - - This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. - The higher 16-bit can be used for storing other data. - \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. -*/ -#ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION -#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) -#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 -#else -#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 -#endif -#endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION - -#if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 -#if RAPIDJSON_64BIT != 1 -#error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 -#endif -#define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) -#define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) -#else -#define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) -#define RAPIDJSON_GETPOINTER(type, p) (p) -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_NEON/RAPIDJSON_SIMD +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD /*! \def RAPIDJSON_SIMD \ingroup RAPIDJSON_CONFIG - \brief Enable SSE2/SSE4.2/Neon optimization. + \brief Enable SSE2/SSE4.2 optimization. RapidJSON supports optimized implementations for some parsing operations - based on the SSE2, SSE4.2 or NEon SIMD extensions on modern Intel - or ARM compatible processors. + based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible + processors. - To enable these optimizations, three different symbols can be defined; + To enable these optimizations, two different symbols can be defined; \code // Enable SSE2 optimization. #define RAPIDJSON_SSE2 @@ -344,17 +281,13 @@ #define RAPIDJSON_SSE42 \endcode - // Enable ARM Neon optimization. - #define RAPIDJSON_NEON - \endcode - - \c RAPIDJSON_SSE42 takes precedence over SSE2, if both are defined. + \c RAPIDJSON_SSE42 takes precedence, if both are defined. If any of these symbols is defined, RapidJSON defines the macro \c RAPIDJSON_SIMD to indicate the availability of the optimized code. */ #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ - || defined(RAPIDJSON_NEON) || defined(RAPIDJSON_DOXYGEN_RUNNING) + || defined(RAPIDJSON_DOXYGEN_RUNNING) #define RAPIDJSON_SIMD #endif @@ -414,33 +347,25 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_STATIC_ASSERT -// Prefer C++11 static_assert, if available +// Adopt from boost #ifndef RAPIDJSON_STATIC_ASSERT -#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) -#define RAPIDJSON_STATIC_ASSERT(x) \ - static_assert(x, RAPIDJSON_STRINGIFY(x)) -#endif // C++11 -#endif // RAPIDJSON_STATIC_ASSERT - -// Adopt C++03 implementation from boost -#ifndef RAPIDJSON_STATIC_ASSERT -#ifndef __clang__ //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#endif RAPIDJSON_NAMESPACE_BEGIN template struct STATIC_ASSERTION_FAILURE; template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; -template struct StaticAssertTest {}; +template struct StaticAssertTest {}; RAPIDJSON_NAMESPACE_END +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y + #if defined(__GNUC__) #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) #else #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif -#ifndef __clang__ //!@endcond -#endif /*! \def RAPIDJSON_STATIC_ASSERT \brief (Internal) macro to check for conditions at compile-time @@ -451,35 +376,6 @@ RAPIDJSON_NAMESPACE_END typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE -#endif // RAPIDJSON_STATIC_ASSERT - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY - -//! Compiler branching hint for expression with high probability to be true. -/*! - \ingroup RAPIDJSON_CONFIG - \param x Boolean expression likely to be true. -*/ -#ifndef RAPIDJSON_LIKELY -#if defined(__GNUC__) || defined(__clang__) -#define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) -#else -#define RAPIDJSON_LIKELY(x) (x) -#endif -#endif - -//! Compiler branching hint for expression with low probability to be true. -/*! - \ingroup RAPIDJSON_CONFIG - \param x Boolean expression unlikely to be true. -*/ -#ifndef RAPIDJSON_UNLIKELY -#if defined(__GNUC__) || defined(__clang__) -#define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define RAPIDJSON_UNLIKELY(x) (x) -#endif #endif /////////////////////////////////////////////////////////////////////////////// @@ -542,12 +438,8 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS #if defined(__clang__) -#if __has_feature(cxx_rvalue_references) && \ +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS __has_feature(cxx_rvalue_references) && \ (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#else -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 -#endif #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1600) @@ -578,17 +470,6 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 #endif -#ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR -#if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ - (defined(_MSC_VER) && _MSC_VER >= 1700) -#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 -#else -#define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 -#endif -#endif // RAPIDJSON_HAS_CXX11_RANGE_FOR - //!@endcond /////////////////////////////////////////////////////////////////////////////// @@ -596,7 +477,7 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_NEW ///! customization point for global \c new -#define RAPIDJSON_NEW(TypeName) new TypeName +#define RAPIDJSON_NEW(x) new x #endif #ifndef RAPIDJSON_DELETE ///! customization point for global \c delete @@ -604,7 +485,10 @@ RAPIDJSON_NAMESPACE_END #endif /////////////////////////////////////////////////////////////////////////////// -// Type +// Allocators and Encodings + +#include "allocators.h" +#include "encodings.h" /*! \namespace rapidjson \brief main RapidJSON namespace @@ -612,6 +496,148 @@ RAPIDJSON_NAMESPACE_END */ RAPIDJSON_NAMESPACE_BEGIN +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + for (size_t i = 0; i < n; i++) + stream.Put(c); +} + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +/////////////////////////////////////////////////////////////////////////////// +// Type + //! Type of JSON value enum Type { kNullType = 0, //!< null diff --git a/contrib/rapidjson/include/rapidjson/reader.h b/contrib/rapidjson/include/rapidjson/reader.h index 120c31115..c5ecf4be5 100644 --- a/contrib/rapidjson/include/rapidjson/reader.h +++ b/contrib/rapidjson/include/rapidjson/reader.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_READER_H_ @@ -17,13 +17,11 @@ /*! \file reader.h */ -#include "allocators.h" -#include "stream.h" -#include "encodedstream.h" +#include "rapidjson.h" +#include "encodings.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strtod.h" -#include #if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) #include @@ -33,8 +31,6 @@ #include #elif defined(RAPIDJSON_SSE2) #include -#elif defined(RAPIDJSON_NEON) -#include #endif #ifdef _MSC_VER @@ -43,13 +39,6 @@ RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant RAPIDJSON_DIAG_OFF(4702) // unreachable code #endif -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(old-style-cast) -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -#endif - #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) @@ -60,7 +49,7 @@ RAPIDJSON_DIAG_OFF(effc++) #ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ - if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ + if (HasParseError()) { return value; } \ RAPIDJSON_MULTILINEMACRO_END #endif #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ @@ -131,7 +120,7 @@ RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // ParseFlag -/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS +/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS \ingroup RAPIDJSON_CONFIG \brief User-defined kParseDefaultFlags definition. @@ -151,10 +140,6 @@ enum ParseFlag { kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). - kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. - kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. - kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. - kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; @@ -163,7 +148,7 @@ enum ParseFlag { /*! \class rapidjson::Handler \brief Concept for receiving events from GenericReader upon parsing. - The functions return true if no error occurs. If they return false, + The functions return true if no error occurs. If they return false, the event publisher should terminate the process. \code concept Handler { @@ -176,8 +161,6 @@ concept Handler { bool Int64(int64_t i); bool Uint64(uint64_t i); bool Double(double d); - /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool RawNumber(const Ch* str, SizeType length, bool copy); bool String(const Ch* str, SizeType length, bool copy); bool StartObject(); bool Key(const Ch* str, SizeType length, bool copy); @@ -208,8 +191,6 @@ struct BaseReaderHandler { bool Int64(int64_t) { return static_cast(*this).Default(); } bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } - /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } @@ -267,17 +248,10 @@ void SkipWhitespace(InputStream& is) { internal::StreamLocalCopy copy(is); InputStream& s(copy.s); - typename InputStream::Ch c; - while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') + while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') s.Take(); } -inline const char* SkipWhitespace(const char* p, const char* end) { - while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - return p; -} - #ifdef RAPIDJSON_SSE42 //! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. inline const char *SkipWhitespace_SIMD(const char* p) { @@ -288,7 +262,7 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; @@ -297,37 +271,23 @@ inline const char *SkipWhitespace_SIMD(const char* p) { // The rest of string using SIMD static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + const __m128i w = _mm_load_si128((const __m128i *)&whitespace[0]); for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); - if (r != 16) // some of characters is non-whitespace - return p + r; + const __m128i s = _mm_load_si128((const __m128i *)p); + const unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) { // some of characters is non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } } } -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; - - // The middle of string using SIMD - static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); - - for (; p <= end - 16; p += 16) { - const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); - const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); - if (r != 16) // some of characters is non-whitespace - return p + r; - } - - return SkipWhitespace(p, end); -} - #elif defined(RAPIDJSON_SSE2) //! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. @@ -339,7 +299,7 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; @@ -347,22 +307,24 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // The rest of string - #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } - static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; - #undef C16 + static const char whitespaces[4][17] = { + " ", + "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r", + "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"}; - const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); - const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); - const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); - const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + const __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]); + const __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]); + const __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]); + const __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]); for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i s = _mm_load_si128((const __m128i *)p); __m128i x = _mm_cmpeq_epi8(s, w0); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = static_cast(~_mm_movemask_epi8(x)); + unsigned short r = (unsigned short)~_mm_movemask_epi8(x); if (r != 0) { // some of characters may be non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; @@ -375,134 +337,11 @@ inline const char *SkipWhitespace_SIMD(const char* p) { } } -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; - - // The rest of string - #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } - static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; - #undef C16 - - const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); - const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); - const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); - const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); - - for (; p <= end - 16; p += 16) { - const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = static_cast(~_mm_movemask_epi8(x)); - if (r != 0) { // some of characters may be non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } - - return SkipWhitespace(p, end); -} - -#elif defined(RAPIDJSON_NEON) - -//! Skip whitespace with ARM Neon instructions, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - const uint8x16_t w0 = vmovq_n_u8(' '); - const uint8x16_t w1 = vmovq_n_u8('\n'); - const uint8x16_t w2 = vmovq_n_u8('\r'); - const uint8x16_t w3 = vmovq_n_u8('\t'); - - for (;; p += 16) { - const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); - uint8x16_t x = vceqq_u8(s, w0); - x = vorrq_u8(x, vceqq_u8(s, w1)); - x = vorrq_u8(x, vceqq_u8(s, w2)); - x = vorrq_u8(x, vceqq_u8(s, w3)); - - x = vmvnq_u8(x); // Negate - x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract - - if (low == 0) { - if (high != 0) { - int lz =__builtin_clzll(high);; - return p + 8 + (lz >> 3); - } - } else { - int lz = __builtin_clzll(low);; - return p + (lz >> 3); - } - } -} - -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; - - const uint8x16_t w0 = vmovq_n_u8(' '); - const uint8x16_t w1 = vmovq_n_u8('\n'); - const uint8x16_t w2 = vmovq_n_u8('\r'); - const uint8x16_t w3 = vmovq_n_u8('\t'); - - for (; p <= end - 16; p += 16) { - const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); - uint8x16_t x = vceqq_u8(s, w0); - x = vorrq_u8(x, vceqq_u8(s, w1)); - x = vorrq_u8(x, vceqq_u8(s, w2)); - x = vorrq_u8(x, vceqq_u8(s, w3)); - - x = vmvnq_u8(x); // Negate - x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract - - if (low == 0) { - if (high != 0) { - int lz = __builtin_clzll(high); - return p + 8 + (lz >> 3); - } - } else { - int lz = __builtin_clzll(low); - return p + (lz >> 3); - } - } - - return SkipWhitespace(p, end); -} - -#endif // RAPIDJSON_NEON +#endif // RAPIDJSON_SSE2 #ifdef RAPIDJSON_SIMD //! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& is) { +template<> inline void SkipWhitespace(InsituStringStream& is) { is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); } @@ -510,27 +349,23 @@ template<> inline void SkipWhitespace(InsituStringStream& is) { template<> inline void SkipWhitespace(StringStream& is) { is.src_ = SkipWhitespace_SIMD(is.src_); } - -template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { - is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); -} #endif // RAPIDJSON_SIMD /////////////////////////////////////////////////////////////////////////////// // GenericReader //! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. -/*! GenericReader parses JSON text from a stream, and send events synchronously to an +/*! GenericReader parses JSON text from a stream, and send events synchronously to an object implementing Handler concept. - It needs to allocate a stack for storing a single decoded string during + It needs to allocate a stack for storing a single decoded string during non-destructive parsing. - For in-situ parsing, the decoded string is directly written to the source + For in-situ parsing, the decoded string is directly written to the source text string, no temporary buffer is required. A GenericReader object can be reused for parsing multiple JSON text. - + \tparam SourceEncoding Encoding of the input stream. \tparam TargetEncoding Encoding of the parse output. \tparam StackAllocator Allocator type for stack. @@ -563,10 +398,9 @@ public: ClearStackOnExit scope(*this); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + SkipWhitespace(is); - if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { + if (is.Peek() == '\0') { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } @@ -575,10 +409,9 @@ public: RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + SkipWhitespace(is); - if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { + if (is.Peek() != '\0') { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } @@ -600,86 +433,9 @@ public: return Parse(is, handler); } - //! Initialize JSON text token-by-token parsing - /*! - */ - void IterativeParseInit() { - parseResult_.Clear(); - state_ = IterativeParsingStartState; - } - - //! Parse one token from JSON text - /*! \tparam InputStream Type of input stream, implementing Stream concept - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template - bool IterativeParseNext(InputStream& is, Handler& handler) { - while (RAPIDJSON_LIKELY(is.Peek() != '\0')) { - SkipWhitespaceAndComments(is); - - Token t = Tokenize(is.Peek()); - IterativeParsingState n = Predict(state_, t); - IterativeParsingState d = Transit(state_, t, n, is, handler); - - // If we've finished or hit an error... - if (RAPIDJSON_UNLIKELY(IsIterativeParsingCompleteState(d))) { - // Report errors. - if (d == IterativeParsingErrorState) { - HandleError(state_, is); - return false; - } - - // Transition to the finish state. - RAPIDJSON_ASSERT(d == IterativeParsingFinishState); - state_ = d; - - // If StopWhenDone is not set... - if (!(parseFlags & kParseStopWhenDoneFlag)) { - // ... and extra non-whitespace data is found... - SkipWhitespaceAndComments(is); - if (is.Peek() != '\0') { - // ... this is considered an error. - HandleError(state_, is); - return false; - } - } - - // Success! We are done! - return true; - } - - // Transition to the new state. - state_ = d; - - // If we parsed anything other than a delimiter, we invoked the handler, so we can return true now. - if (!IsIterativeParsingDelimiterState(n)) - return true; - } - - // We reached the end of file. - stack_.Clear(); - - if (state_ != IterativeParsingFinishState) { - HandleError(state_, is); - return false; - } - - return true; - } - - //! Check if token-by-token parsing JSON text is complete - /*! \return Whether the JSON has been fully decoded. - */ - RAPIDJSON_FORCEINLINE bool IterativeParseComplete() { - return IsIterativeParsingCompleteState(state_); - } - //! Whether a parse error has occured in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } - + //! Get the \ref ParseErrorCode of last parsing. ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } @@ -706,98 +462,52 @@ private: ClearStackOnExit& operator=(const ClearStackOnExit&); }; - template - void SkipWhitespaceAndComments(InputStream& is) { - SkipWhitespace(is); - - if (parseFlags & kParseCommentsFlag) { - while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { - if (Consume(is, '*')) { - while (true) { - if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - else if (Consume(is, '*')) { - if (Consume(is, '/')) - break; - } - else - is.Take(); - } - } - else if (RAPIDJSON_LIKELY(Consume(is, '/'))) - while (is.Peek() != '\0' && is.Take() != '\n') {} - else - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - - SkipWhitespace(is); - } - } - } - // Parse object: { string : value, ... } template void ParseObject(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '{'); is.Take(); // Skip '{' - - if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + + if (!handler.StartObject()) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespace(is); - if (Consume(is, '}')) { - if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object + if (is.Peek() == '}') { + is.Take(); + if (!handler.EndObject(0)) // empty object RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } for (SizeType memberCount = 0;;) { - if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) + if (is.Peek() != '"') RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); ParseString(is, handler, true); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespace(is); - if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) + if (is.Take() != ':') RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespace(is); ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespace(is); ++memberCount; - switch (is.Peek()) { - case ',': - is.Take(); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - break; - case '}': - is.Take(); - if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + switch (is.Take()) { + case ',': SkipWhitespace(is); break; + case '}': + if (!handler.EndObject(memberCount)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; - default: - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy - } - - if (parseFlags & kParseTrailingCommasFlag) { - if (is.Peek() == '}') { - if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - is.Take(); - return; - } + default: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); } } } @@ -807,15 +517,15 @@ private: void ParseArray(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '['); is.Take(); // Skip '[' - - if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + + if (!handler.StartArray()) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespace(is); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - if (Consume(is, ']')) { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array + if (is.Peek() == ']') { + is.Take(); + if (!handler.EndArray(0)) // empty array RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } @@ -825,28 +535,15 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++elementCount; - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespace(is); - if (Consume(is, ',')) { - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - } - else if (Consume(is, ']')) { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - - if (parseFlags & kParseTrailingCommasFlag) { - if (is.Peek() == ']') { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + switch (is.Take()) { + case ',': SkipWhitespace(is); break; + case ']': + if (!handler.EndArray(elementCount)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - is.Take(); return; - } + default: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); } } } @@ -856,12 +553,12 @@ private: RAPIDJSON_ASSERT(is.Peek() == 'n'); is.Take(); - if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { - if (RAPIDJSON_UNLIKELY(!handler.Null())) + if (is.Take() == 'u' && is.Take() == 'l' && is.Take() == 'l') { + if (!handler.Null()) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); } template @@ -869,12 +566,12 @@ private: RAPIDJSON_ASSERT(is.Peek() == 't'); is.Take(); - if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { - if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) + if (is.Take() == 'r' && is.Take() == 'u' && is.Take() == 'e') { + if (!handler.Bool(true)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); } template @@ -882,30 +579,20 @@ private: RAPIDJSON_ASSERT(is.Peek() == 'f'); is.Take(); - if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { - if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) + if (is.Take() == 'a' && is.Take() == 'l' && is.Take() == 's' && is.Take() == 'e') { + if (!handler.Bool(false)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); - } - - template - RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { - if (RAPIDJSON_LIKELY(is.Peek() == expect)) { - is.Take(); - return true; - } - else - return false; + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); } // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). template - unsigned ParseHex4(InputStream& is, size_t escapeOffset) { + unsigned ParseHex4(InputStream& is) { unsigned codepoint = 0; for (int i = 0; i < 4; i++) { - Ch c = is.Peek(); + Ch c = is.Take(); codepoint <<= 4; codepoint += static_cast(c); if (c >= '0' && c <= '9') @@ -915,10 +602,9 @@ private: else if (c >= 'a' && c <= 'f') codepoint -= 'a' - 10; else { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, is.Tell() - 1); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); } - is.Take(); } return codepoint; } @@ -933,14 +619,7 @@ private: *stack_.template Push() = c; ++length_; } - - RAPIDJSON_FORCEINLINE void* Push(SizeType count) { - length_ += count; - return stack_.template Push(count); - } - size_t Length() const { return length_; } - Ch* Pop() { return stack_.template Pop(length_); } @@ -959,9 +638,6 @@ private: internal::StreamLocalCopy copy(is); InputStream& s(copy.s); - RAPIDJSON_ASSERT(s.Peek() == '\"'); - s.Take(); // Skip '\"' - bool success = false; if (parseFlags & kParseInsituFlag) { typename InputStream::Ch *head = s.PutBegin(); @@ -969,7 +645,7 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; size_t length = s.PutEnd(head) - 1; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + const typename TargetEncoding::Ch* const str = (typename TargetEncoding::Ch*)head; success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); } else { @@ -980,7 +656,7 @@ private: const typename TargetEncoding::Ch* const str = stackStream.Pop(); success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); } - if (RAPIDJSON_UNLIKELY(!success)) + if (!success) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); } @@ -991,421 +667,74 @@ private: //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 }; #undef Z16 //!@endcond - for (;;) { - // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. - if (!(parseFlags & kParseValidateEncodingFlag)) - ScanCopyUnescapedString(is, os); + RAPIDJSON_ASSERT(is.Peek() == '\"'); + is.Take(); // Skip '\"' + for (;;) { Ch c = is.Peek(); - if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape - size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset + if (c == '\\') { // Escape is.Take(); - Ch e = is.Peek(); - if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { - is.Take(); - os.Put(static_cast(escape[static_cast(e)])); + Ch e = is.Take(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { + os.Put(escape[(unsigned char)e]); } - else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode - is.Take(); - unsigned codepoint = ParseHex4(is, escapeOffset); + else if (e == 'u') { // Unicode + unsigned codepoint = ParseHex4(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { // Handle UTF-16 surrogate pair - if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - unsigned codepoint2 = ParseHex4(is, escapeOffset); + if (is.Take() != '\\' || is.Take() != 'u') + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); + unsigned codepoint2 = ParseHex4(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; } TEncoding::Encode(os, codepoint); } else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); } - else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote + else if (c == '"') { // Closing double quote is.Take(); os.Put('\0'); // null-terminate the string return; } - else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - if (c == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); - else + else if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); + else if ((unsigned)c < 0x20) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + else { + if (parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)) RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); } - else { - size_t offset = is.Tell(); - if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? - !Transcoder::Validate(is, os) : - !Transcoder::Transcode(is, os)))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); - } } } - template - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { - // Do nothing for generic version - } - -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) - // StringStream -> StackStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { - const char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = p; - return; - } - else - os.Put(*p++); - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - SizeType length; - #ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; - #else - length = static_cast(__builtin_ffs(r) - 1); - #endif - if (length != 0) { - char* q = reinterpret_cast(os.Push(length)); - for (size_t i = 0; i < length; i++) - q[i] = p[i]; - - p += length; - } - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); - } - - is.src_ = p; - } - - // InsituStringStream -> InsituStringStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { - RAPIDJSON_ASSERT(&is == &os); - (void)os; - - if (is.src_ == is.dst_) { - SkipUnescapedString(is); - return; - } - - char* p = is.src_; - char *q = is.dst_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = p; - is.dst_ = q; - return; - } - else - *q++ = *p++; - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (;; p += 16, q += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - size_t length; -#ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; -#else - length = static_cast(__builtin_ffs(r) - 1); -#endif - for (const char* pend = p + length; p != pend; ) - *q++ = *p++; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); - } - - is.src_ = p; - is.dst_ = q; - } - - // When read/write pointers are the same for insitu stream, just skip unescaped characters - static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { - RAPIDJSON_ASSERT(is.src_ == is.dst_); - char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - for (; p != nextAligned; p++) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = is.dst_ = p; - return; - } - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - size_t length; -#ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; -#else - length = static_cast(__builtin_ffs(r) - 1); -#endif - p += length; - break; - } - } - - is.src_ = is.dst_ = p; - } -#elif defined(RAPIDJSON_NEON) - // StringStream -> StackStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { - const char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = p; - return; - } - else - os.Put(*p++); - - // The rest of string using SIMD - const uint8x16_t s0 = vmovq_n_u8('"'); - const uint8x16_t s1 = vmovq_n_u8('\\'); - const uint8x16_t s2 = vmovq_n_u8('\b'); - const uint8x16_t s3 = vmovq_n_u8(32); - - for (;; p += 16) { - const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); - uint8x16_t x = vceqq_u8(s, s0); - x = vorrq_u8(x, vceqq_u8(s, s1)); - x = vorrq_u8(x, vceqq_u8(s, s2)); - x = vorrq_u8(x, vcltq_u8(s, s3)); - - x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract - - SizeType length = 0; - bool escaped = false; - if (low == 0) { - if (high != 0) { - unsigned lz = (unsigned)__builtin_clzll(high);; - length = 8 + (lz >> 3); - escaped = true; - } - } else { - unsigned lz = (unsigned)__builtin_clzll(low);; - length = lz >> 3; - escaped = true; - } - if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped - if (length != 0) { - char* q = reinterpret_cast(os.Push(length)); - for (size_t i = 0; i < length; i++) - q[i] = p[i]; - - p += length; - } - break; - } - vst1q_u8(reinterpret_cast(os.Push(16)), s); - } - - is.src_ = p; - } - - // InsituStringStream -> InsituStringStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { - RAPIDJSON_ASSERT(&is == &os); - (void)os; - - if (is.src_ == is.dst_) { - SkipUnescapedString(is); - return; - } - - char* p = is.src_; - char *q = is.dst_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = p; - is.dst_ = q; - return; - } - else - *q++ = *p++; - - // The rest of string using SIMD - const uint8x16_t s0 = vmovq_n_u8('"'); - const uint8x16_t s1 = vmovq_n_u8('\\'); - const uint8x16_t s2 = vmovq_n_u8('\b'); - const uint8x16_t s3 = vmovq_n_u8(32); - - for (;; p += 16, q += 16) { - const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); - uint8x16_t x = vceqq_u8(s, s0); - x = vorrq_u8(x, vceqq_u8(s, s1)); - x = vorrq_u8(x, vceqq_u8(s, s2)); - x = vorrq_u8(x, vcltq_u8(s, s3)); - - x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract - - SizeType length = 0; - bool escaped = false; - if (low == 0) { - if (high != 0) { - unsigned lz = (unsigned)__builtin_clzll(high); - length = 8 + (lz >> 3); - escaped = true; - } - } else { - unsigned lz = (unsigned)__builtin_clzll(low); - length = lz >> 3; - escaped = true; - } - if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped - for (const char* pend = p + length; p != pend; ) { - *q++ = *p++; - } - break; - } - vst1q_u8(reinterpret_cast(q), s); - } - - is.src_ = p; - is.dst_ = q; - } - - // When read/write pointers are the same for insitu stream, just skip unescaped characters - static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { - RAPIDJSON_ASSERT(is.src_ == is.dst_); - char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - for (; p != nextAligned; p++) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = is.dst_ = p; - return; - } - - // The rest of string using SIMD - const uint8x16_t s0 = vmovq_n_u8('"'); - const uint8x16_t s1 = vmovq_n_u8('\\'); - const uint8x16_t s2 = vmovq_n_u8('\b'); - const uint8x16_t s3 = vmovq_n_u8(32); - - for (;; p += 16) { - const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); - uint8x16_t x = vceqq_u8(s, s0); - x = vorrq_u8(x, vceqq_u8(s, s1)); - x = vorrq_u8(x, vceqq_u8(s, s2)); - x = vorrq_u8(x, vcltq_u8(s, s3)); - - x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract - - if (low == 0) { - if (high != 0) { - int lz = __builtin_clzll(high); - p += 8 + (lz >> 3); - break; - } - } else { - int lz = __builtin_clzll(low); - p += lz >> 3; - break; - } - } - - is.src_ = is.dst_ = p; - } -#endif // RAPIDJSON_NEON - - template + template class NumberStream; template - class NumberStream { + class NumberStream { public: - typedef typename InputStream::Ch Ch; - NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } + ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char) {} - size_t Tell() { return is.Tell(); } size_t Length() { return 0; } const char* Pop() { return 0; } @@ -1417,20 +746,17 @@ private: }; template - class NumberStream : public NumberStream { - typedef NumberStream Base; + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} + NumberStream(GenericReader& reader, InputStream& is) : NumberStream(reader, is), stackStream(reader.stack_) {} + ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put(static_cast(Base::is.Peek())); + stackStream.Put((char)Base::is.Peek()); return Base::is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char c) { - stackStream.Put(c); - } - size_t Length() { return stackStream.Length(); } const char* Pop() { @@ -1442,48 +768,34 @@ private: StackStream stackStream; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; - public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} - - RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } - }; - template void ParseNumber(InputStream& is, Handler& handler) { internal::StreamLocalCopy copy(is); - NumberStream s(*this, copy.s); - - size_t startOffset = s.Tell(); - double d = 0.0; - bool useNanOrInf = false; + NumberStream s(*this, copy.s); // Parse minus - bool minus = Consume(s, '-'); + bool minus = false; + if (s.Peek() == '-') { + minus = true; + s.Take(); + } // Parse int: zero / ( digit1-9 *DIGIT ) unsigned i = 0; uint64_t i64 = 0; bool use64bit = false; int significandDigit = 0; - if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { + if (s.Peek() == '0') { i = 0; s.TakePush(); } - else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { + else if (s.Peek() >= '1' && s.Peek() <= '9') { i = static_cast(s.TakePush() - '0'); if (minus) - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 - if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i >= 214748364) { // 2^31 = 2147483648 + if (i != 214748364 || s.Peek() > '8') { i64 = i; use64bit = true; break; @@ -1493,9 +805,9 @@ private: significandDigit++; } else - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 - if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i >= 429496729) { // 2^32 - 1 = 4294967295 + if (i != 429496729 || s.Peek() > '5') { i64 = i; use64bit = true; break; @@ -1505,41 +817,18 @@ private: significandDigit++; } } - // Parse NaN or Infinity here - else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { - if (Consume(s, 'N')) { - if (Consume(s, 'a') && Consume(s, 'N')) { - d = std::numeric_limits::quiet_NaN(); - useNanOrInf = true; - } - } - else if (RAPIDJSON_LIKELY(Consume(s, 'I'))) { - if (Consume(s, 'n') && Consume(s, 'f')) { - d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); - useNanOrInf = true; - - if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') - && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) { - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - } - } - } - - if (RAPIDJSON_UNLIKELY(!useNanOrInf)) { - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - } - } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); // Parse 64bit int bool useDouble = false; + double d = 0.0; if (use64bit) { - if (minus) - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 - if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { - d = static_cast(i64); + if (minus) + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC)) // 2^63 = 9223372036854775808 + if (i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8') { + d = i64; useDouble = true; break; } @@ -1547,10 +836,10 @@ private: significandDigit++; } else - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 - if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { - d = static_cast(i64); + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) // 2^64 - 1 = 18446744073709551615 + if (i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5') { + d = i64; useDouble = true; break; } @@ -1561,9 +850,9 @@ private: // Force double for big integer if (useDouble) { - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + while (s.Peek() >= '0' && s.Peek() <= '9') { + if (d >= 1.7976931348623157e307) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); d = d * 10 + (s.TakePush() - '0'); } } @@ -1571,10 +860,11 @@ private: // Parse frac = decimal-point 1*DIGIT int expFrac = 0; size_t decimalPosition; - if (Consume(s, '.')) { + if (s.Peek() == '.') { + s.Take(); decimalPosition = s.Length(); - if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) + if (!(s.Peek() >= '0' && s.Peek() <= '9')) RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); if (!useDouble) { @@ -1582,8 +872,8 @@ private: // Use i64 to store significand in 64-bit architecture if (!use64bit) i64 = i; - - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + + while (s.Peek() >= '0' && s.Peek() <= '9') { if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path break; else { @@ -1594,19 +884,19 @@ private: } } - d = static_cast(i64); + d = (double)i64; #else // Use double to store significand in 32-bit architecture - d = static_cast(use64bit ? i64 : i); + d = use64bit ? (double)i64 : (double)i; #endif useDouble = true; } - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + while (s.Peek() >= '0' && s.Peek() <= '9') { if (significandDigit < 17) { d = d * 10.0 + (s.TakePush() - '0'); --expFrac; - if (RAPIDJSON_LIKELY(d > 0.0)) + if (d > 0.0) significandDigit++; } else @@ -1618,35 +908,38 @@ private: // Parse exp = e [ minus / plus ] 1*DIGIT int exp = 0; - if (Consume(s, 'e') || Consume(s, 'E')) { + if (s.Peek() == 'e' || s.Peek() == 'E') { if (!useDouble) { - d = static_cast(use64bit ? i64 : i); + d = use64bit ? i64 : i; useDouble = true; } + s.Take(); bool expMinus = false; - if (Consume(s, '+')) - ; - else if (Consume(s, '-')) + if (s.Peek() == '+') + s.Take(); + else if (s.Peek() == '-') { + s.Take(); expMinus = true; + } - if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = static_cast(s.Take() - '0'); + if (s.Peek() >= '0' && s.Peek() <= '9') { + exp = s.Take() - '0'; if (expMinus) { - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = exp * 10 + static_cast(s.Take() - '0'); + while (s.Peek() >= '0' && s.Peek() <= '9') { + exp = exp * 10 + (s.Take() - '0'); if (exp >= 214748364) { // Issue #313: prevent overflow exponent - while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent + while (s.Peek() >= '0' && s.Peek() <= '9') // Consume the rest of exponent s.Take(); } } } else { // positive exp int maxExp = 308 - expFrac; - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = exp * 10 + static_cast(s.Take() - '0'); - if (RAPIDJSON_UNLIKELY(exp > maxExp)) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + while (s.Peek() >= '0' && s.Peek() <= '9') { + exp = exp * 10 + (s.Take() - '0'); + if (exp > maxExp) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); } } } @@ -1659,63 +952,34 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - if (parseFlags & kParseNumbersAsStringsFlag) { - if (parseFlags & kParseInsituFlag) { - s.Pop(); // Pop stack no matter if it will be used or not. - typename InputStream::Ch* head = is.PutBegin(); - const size_t length = s.Tell() - startOffset; - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - // unable to insert the \0 character here, it will erase the comma after this number - const typename TargetEncoding::Ch* const str = reinterpret_cast(head); - cont = handler.RawNumber(str, SizeType(length), false); - } - else { - SizeType numCharsToCopy = static_cast(s.Length()); - StringStream srcStream(s.Pop()); - StackStream dstStream(stack_); - while (numCharsToCopy--) { - Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); - } - dstStream.Put('\0'); - const typename TargetEncoding::Ch* str = dstStream.Pop(); - const SizeType length = static_cast(dstStream.Length()) - 1; - cont = handler.RawNumber(str, SizeType(length), true); - } + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); } else { - size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - - if (useDouble) { - int p = exp + expFrac; - if (parseFlags & kParseFullPrecisionFlag) - d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); - else - d = internal::StrtodNormalPrecision(d, p); - - cont = handler.Double(minus ? -d : d); - } - else if (useNanOrInf) { - cont = handler.Double(d); - } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(static_cast(~i64 + 1)); - else - cont = handler.Uint64(i64); - } - else { - if (minus) - cont = handler.Int(static_cast(~i + 1)); - else - cont = handler.Uint(i); - } - } + if (use64bit) { + if (minus) + cont = handler.Int64(-(int64_t)i64); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(-(int)i); + else + cont = handler.Uint(i); + } } - if (RAPIDJSON_UNLIKELY(!cont)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); + if (!cont) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); } // Parse any JSON value @@ -1728,10 +992,7 @@ private: case '"': ParseString(is, handler); break; case '{': ParseObject(is, handler); break; case '[': ParseArray (is, handler); break; - default : - ParseNumber(is, handler); - break; - + default : ParseNumber(is, handler); } } @@ -1739,29 +1000,27 @@ private: // States enum IterativeParsingState { - IterativeParsingFinishState = 0, // sink states at top - IterativeParsingErrorState, // sink states at top - IterativeParsingStartState, + IterativeParsingStartState = 0, + IterativeParsingFinishState, + IterativeParsingErrorState, // Object states IterativeParsingObjectInitialState, IterativeParsingMemberKeyState, + IterativeParsingKeyValueDelimiterState, IterativeParsingMemberValueState, + IterativeParsingMemberDelimiterState, IterativeParsingObjectFinishState, // Array states IterativeParsingArrayInitialState, IterativeParsingElementState, + IterativeParsingElementDelimiterState, IterativeParsingArrayFinishState, // Single value state IterativeParsingValueState, - - // Delimiter states (at bottom) - IterativeParsingElementDelimiterState, - IterativeParsingMemberDelimiterState, - IterativeParsingKeyValueDelimiterState, - + cIterativeParsingStateCount }; @@ -1805,9 +1064,9 @@ private: #undef N #undef N16 //!@endcond - - if (sizeof(Ch) == 1 || static_cast(c) < 256) - return static_cast(tokenMap[static_cast(c)]); + + if (sizeof(Ch) == 1 || unsigned(c) < 256) + return (Token)tokenMap[(unsigned char)c]; else return NumberToken; } @@ -1815,18 +1074,6 @@ private: RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { // current state x one lookahead token -> new state static const char G[cIterativeParsingStateCount][kTokenCount] = { - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, // Start { IterativeParsingArrayInitialState, // Left bracket @@ -1841,6 +1088,18 @@ private: IterativeParsingValueState, // Null IterativeParsingValueState // Number }, + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, // ObjectInitial { IterativeParsingErrorState, // Left bracket @@ -1869,6 +1128,20 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, // MemberValue { IterativeParsingErrorState, // Left bracket @@ -1883,6 +1156,20 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, // ObjectFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1917,6 +1204,20 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, // ArrayFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1928,52 +1229,10 @@ private: IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState - }, - // ElementDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, + } }; // End of G - return static_cast(G[state][token]); + return (IterativeParsingState)G[state][token]; } // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). @@ -2050,11 +1309,6 @@ private: case IterativeParsingObjectFinishState: { - // Transit from delimiter is only allowed when trailing commas are enabled - if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); - return IterativeParsingErrorState; - } // Get member count. SizeType c = *stack_.template Pop(1); // If the object is not empty, count the last member. @@ -2080,11 +1334,6 @@ private: case IterativeParsingArrayFinishState: { - // Transit from delimiter is only allowed when trailing commas are enabled - if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); - return IterativeParsingErrorState; - } // Get element count. SizeType c = *stack_.template Pop(1); // If the array is not empty, count the last element. @@ -2136,68 +1385,55 @@ private: // Error flag has been set. return; } - + switch (src) { - case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; - case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); case IterativeParsingObjectInitialState: - case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; - case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; - case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; - case IterativeParsingKeyValueDelimiterState: - case IterativeParsingArrayInitialState: - case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; - default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; - } + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); + case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + } } - RAPIDJSON_FORCEINLINE bool IsIterativeParsingDelimiterState(IterativeParsingState s) { - return s >= IterativeParsingElementDelimiterState; - } - - RAPIDJSON_FORCEINLINE bool IsIterativeParsingCompleteState(IterativeParsingState s) { - return s <= IterativeParsingErrorState; - } - template ParseResult IterativeParse(InputStream& is, Handler& handler) { parseResult_.Clear(); ClearStackOnExit scope(*this); IterativeParsingState state = IterativeParsingStartState; - - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + SkipWhitespace(is); while (is.Peek() != '\0') { Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); IterativeParsingState d = Transit(state, t, n, is, handler); - + if (d == IterativeParsingErrorState) { HandleError(state, is); break; } - + state = d; - + // Do not further consume streams if a root JSON has been parsed. if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) break; - - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + SkipWhitespace(is); } - + // Handle the end of file. if (state != IterativeParsingFinishState) HandleError(state, is); - + return parseResult_; } static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. ParseResult parseResult_; - IterativeParsingState state_; }; // class GenericReader //! Reader with UTF8 encoding and default allocator. @@ -2205,11 +1441,6 @@ typedef GenericReader, UTF8<> > Reader; RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - - #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/schema.h b/contrib/rapidjson/include/rapidjson/schema.h deleted file mode 100644 index abcf1a102..000000000 --- a/contrib/rapidjson/include/rapidjson/schema.h +++ /dev/null @@ -1,2016 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available-> -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License-> You may obtain a copy of the License at -// -// http://opensource->org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied-> See the License for the -// specific language governing permissions and limitations under the License-> - -#ifndef RAPIDJSON_SCHEMA_H_ -#define RAPIDJSON_SCHEMA_H_ - -#include "document.h" -#include "pointer.h" -#include // abs, floor - -#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 -#endif - -#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) -#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 -#endif - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX -#include "internal/regex.h" -#elif RAPIDJSON_SCHEMA_USE_STDREGEX -#include -#endif - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX -#define RAPIDJSON_SCHEMA_HAS_REGEX 1 -#else -#define RAPIDJSON_SCHEMA_HAS_REGEX 0 -#endif - -#ifndef RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_VERBOSE 0 -#endif - -#if RAPIDJSON_SCHEMA_VERBOSE -#include "stringbuffer.h" -#endif - -RAPIDJSON_DIAG_PUSH - -#if defined(__GNUC__) -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_OFF(weak-vtables) -RAPIDJSON_DIAG_OFF(exit-time-destructors) -RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) -RAPIDJSON_DIAG_OFF(variadic-macros) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Verbose Utilities - -#if RAPIDJSON_SCHEMA_VERBOSE - -namespace internal { - -inline void PrintInvalidKeyword(const char* keyword) { - printf("Fail keyword: %s\n", keyword); -} - -inline void PrintInvalidKeyword(const wchar_t* keyword) { - wprintf(L"Fail keyword: %ls\n", keyword); -} - -inline void PrintInvalidDocument(const char* document) { - printf("Fail document: %s\n\n", document); -} - -inline void PrintInvalidDocument(const wchar_t* document) { - wprintf(L"Fail document: %ls\n\n", document); -} - -inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { - printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); -} - -inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { - wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); -} - -} // namespace internal - -#endif // RAPIDJSON_SCHEMA_VERBOSE - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_INVALID_KEYWORD_RETURN - -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) -#else -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) -#endif - -#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - context.invalidKeyword = keyword.GetString();\ - RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ - return false;\ -RAPIDJSON_MULTILINEMACRO_END - -/////////////////////////////////////////////////////////////////////////////// -// Forward declarations - -template -class GenericSchemaDocument; - -namespace internal { - -template -class Schema; - -/////////////////////////////////////////////////////////////////////////////// -// ISchemaValidator - -class ISchemaValidator { -public: - virtual ~ISchemaValidator() {} - virtual bool IsValid() const = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// ISchemaStateFactory - -template -class ISchemaStateFactory { -public: - virtual ~ISchemaStateFactory() {} - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; - virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; - virtual void* CreateHasher() = 0; - virtual uint64_t GetHashCode(void* hasher) = 0; - virtual void DestroryHasher(void* hasher) = 0; - virtual void* MallocState(size_t size) = 0; - virtual void FreeState(void* p) = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// Hasher - -// For comparison of compound value -template -class Hasher { -public: - typedef typename Encoding::Ch Ch; - - Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} - - bool Null() { return WriteType(kNullType); } - bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } - bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } - bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } - bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } - bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } - bool Double(double d) { - Number n; - if (d < 0) n.u.i = static_cast(d); - else n.u.u = static_cast(d); - n.d = d; - return WriteNumber(n); - } - - bool RawNumber(const Ch* str, SizeType len, bool) { - WriteBuffer(kNumberType, str, len * sizeof(Ch)); - return true; - } - - bool String(const Ch* str, SizeType len, bool) { - WriteBuffer(kStringType, str, len * sizeof(Ch)); - return true; - } - - bool StartObject() { return true; } - bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } - bool EndObject(SizeType memberCount) { - uint64_t h = Hash(0, kObjectType); - uint64_t* kv = stack_.template Pop(memberCount * 2); - for (SizeType i = 0; i < memberCount; i++) - h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive - *stack_.template Push() = h; - return true; - } - - bool StartArray() { return true; } - bool EndArray(SizeType elementCount) { - uint64_t h = Hash(0, kArrayType); - uint64_t* e = stack_.template Pop(elementCount); - for (SizeType i = 0; i < elementCount; i++) - h = Hash(h, e[i]); // Use hash to achieve element order sensitive - *stack_.template Push() = h; - return true; - } - - bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } - - uint64_t GetHashCode() const { - RAPIDJSON_ASSERT(IsValid()); - return *stack_.template Top(); - } - -private: - static const size_t kDefaultSize = 256; - struct Number { - union U { - uint64_t u; - int64_t i; - }u; - double d; - }; - - bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } - - bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } - - bool WriteBuffer(Type type, const void* data, size_t len) { - // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ - uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); - const unsigned char* d = static_cast(data); - for (size_t i = 0; i < len; i++) - h = Hash(h, d[i]); - *stack_.template Push() = h; - return true; - } - - static uint64_t Hash(uint64_t h, uint64_t d) { - static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); - h ^= d; - h *= kPrime; - return h; - } - - Stack stack_; -}; - -/////////////////////////////////////////////////////////////////////////////// -// SchemaValidationContext - -template -struct SchemaValidationContext { - typedef Schema SchemaType; - typedef ISchemaStateFactory SchemaValidatorFactoryType; - typedef typename SchemaType::ValueType ValueType; - typedef typename ValueType::Ch Ch; - - enum PatternValidatorType { - kPatternValidatorOnly, - kPatternValidatorWithProperty, - kPatternValidatorWithAdditionalProperty - }; - - SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : - factory(f), - schema(s), - valueSchema(), - invalidKeyword(), - hasher(), - arrayElementHashCodes(), - validators(), - validatorCount(), - patternPropertiesValidators(), - patternPropertiesValidatorCount(), - patternPropertiesSchemas(), - patternPropertiesSchemaCount(), - valuePatternValidatorType(kPatternValidatorOnly), - propertyExist(), - inArray(false), - valueUniqueness(false), - arrayUniqueness(false) - { - } - - ~SchemaValidationContext() { - if (hasher) - factory.DestroryHasher(hasher); - if (validators) { - for (SizeType i = 0; i < validatorCount; i++) - factory.DestroySchemaValidator(validators[i]); - factory.FreeState(validators); - } - if (patternPropertiesValidators) { - for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) - factory.DestroySchemaValidator(patternPropertiesValidators[i]); - factory.FreeState(patternPropertiesValidators); - } - if (patternPropertiesSchemas) - factory.FreeState(patternPropertiesSchemas); - if (propertyExist) - factory.FreeState(propertyExist); - } - - SchemaValidatorFactoryType& factory; - const SchemaType* schema; - const SchemaType* valueSchema; - const Ch* invalidKeyword; - void* hasher; // Only validator access - void* arrayElementHashCodes; // Only validator access this - ISchemaValidator** validators; - SizeType validatorCount; - ISchemaValidator** patternPropertiesValidators; - SizeType patternPropertiesValidatorCount; - const SchemaType** patternPropertiesSchemas; - SizeType patternPropertiesSchemaCount; - PatternValidatorType valuePatternValidatorType; - PatternValidatorType objectPatternValidatorType; - SizeType arrayElementIndex; - bool* propertyExist; - bool inArray; - bool valueUniqueness; - bool arrayUniqueness; -}; - -/////////////////////////////////////////////////////////////////////////////// -// Schema - -template -class Schema { -public: - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename SchemaDocumentType::AllocatorType AllocatorType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef SchemaValidationContext Context; - typedef Schema SchemaType; - typedef GenericValue SValue; - friend class GenericSchemaDocument; - - Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : - allocator_(allocator), - typeless_(schemaDocument->GetTypeless()), - enum_(), - enumCount_(), - not_(), - type_((1 << kTotalSchemaType) - 1), // typeless - validatorCount_(), - properties_(), - additionalPropertiesSchema_(), - patternProperties_(), - patternPropertyCount_(), - propertyCount_(), - minProperties_(), - maxProperties_(SizeType(~0)), - additionalProperties_(true), - hasDependencies_(), - hasRequired_(), - hasSchemaDependencies_(), - additionalItemsSchema_(), - itemsList_(), - itemsTuple_(), - itemsTupleCount_(), - minItems_(), - maxItems_(SizeType(~0)), - additionalItems_(true), - uniqueItems_(false), - pattern_(), - minLength_(0), - maxLength_(~SizeType(0)), - exclusiveMinimum_(false), - exclusiveMaximum_(false) - { - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename ValueType::ConstValueIterator ConstValueIterator; - typedef typename ValueType::ConstMemberIterator ConstMemberIterator; - - if (!value.IsObject()) - return; - - if (const ValueType* v = GetMember(value, GetTypeString())) { - type_ = 0; - if (v->IsString()) - AddType(*v); - else if (v->IsArray()) - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) - AddType(*itr); - } - - if (const ValueType* v = GetMember(value, GetEnumString())) - if (v->IsArray() && v->Size() > 0) { - enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { - typedef Hasher > EnumHasherType; - char buffer[256 + 24]; - MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); - EnumHasherType h(&hasherAllocator, 256); - itr->Accept(h); - enum_[enumCount_++] = h.GetHashCode(); - } - } - - if (schemaDocument) { - AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); - AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); - AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); - } - - if (const ValueType* v = GetMember(value, GetNotString())) { - schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); - notValidatorIndex_ = validatorCount_; - validatorCount_++; - } - - // Object - - const ValueType* properties = GetMember(value, GetPropertiesString()); - const ValueType* required = GetMember(value, GetRequiredString()); - const ValueType* dependencies = GetMember(value, GetDependenciesString()); - { - // Gather properties from properties/required/dependencies - SValue allProperties(kArrayType); - - if (properties && properties->IsObject()) - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) - AddUniqueElement(allProperties, itr->name); - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) - AddUniqueElement(allProperties, *itr); - - if (dependencies && dependencies->IsObject()) - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - AddUniqueElement(allProperties, itr->name); - if (itr->value.IsArray()) - for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) - if (i->IsString()) - AddUniqueElement(allProperties, *i); - } - - if (allProperties.Size() > 0) { - propertyCount_ = allProperties.Size(); - properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); - for (SizeType i = 0; i < propertyCount_; i++) { - new (&properties_[i]) Property(); - properties_[i].name = allProperties[i]; - properties_[i].schema = typeless_; - } - } - } - - if (properties && properties->IsObject()) { - PointerType q = p.Append(GetPropertiesString(), allocator_); - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { - SizeType index; - if (FindPropertyIndex(itr->name, &index)) - schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); - } - } - - if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { - PointerType q = p.Append(GetPatternPropertiesString(), allocator_); - patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); - patternPropertyCount_ = 0; - - for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { - new (&patternProperties_[patternPropertyCount_]) PatternProperty(); - patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); - schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); - patternPropertyCount_++; - } - } - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) { - SizeType index; - if (FindPropertyIndex(*itr, &index)) { - properties_[index].required = true; - hasRequired_ = true; - } - } - - if (dependencies && dependencies->IsObject()) { - PointerType q = p.Append(GetDependenciesString(), allocator_); - hasDependencies_ = true; - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - SizeType sourceIndex; - if (FindPropertyIndex(itr->name, &sourceIndex)) { - if (itr->value.IsArray()) { - properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); - std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); - for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { - SizeType targetIndex; - if (FindPropertyIndex(*targetItr, &targetIndex)) - properties_[sourceIndex].dependencies[targetIndex] = true; - } - } - else if (itr->value.IsObject()) { - hasSchemaDependencies_ = true; - schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); - properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; - validatorCount_++; - } - } - } - } - - if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { - if (v->IsBool()) - additionalProperties_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); - } - - AssignIfExist(minProperties_, value, GetMinPropertiesString()); - AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); - - // Array - if (const ValueType* v = GetMember(value, GetItemsString())) { - PointerType q = p.Append(GetItemsString(), allocator_); - if (v->IsObject()) // List validation - schemaDocument->CreateSchema(&itemsList_, q, *v, document); - else if (v->IsArray()) { // Tuple validation - itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); - SizeType index = 0; - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) - schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); - } - } - - AssignIfExist(minItems_, value, GetMinItemsString()); - AssignIfExist(maxItems_, value, GetMaxItemsString()); - - if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { - if (v->IsBool()) - additionalItems_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); - } - - AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); - - // String - AssignIfExist(minLength_, value, GetMinLengthString()); - AssignIfExist(maxLength_, value, GetMaxLengthString()); - - if (const ValueType* v = GetMember(value, GetPatternString())) - pattern_ = CreatePattern(*v); - - // Number - if (const ValueType* v = GetMember(value, GetMinimumString())) - if (v->IsNumber()) - minimum_.CopyFrom(*v, *allocator_); - - if (const ValueType* v = GetMember(value, GetMaximumString())) - if (v->IsNumber()) - maximum_.CopyFrom(*v, *allocator_); - - AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); - AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); - - if (const ValueType* v = GetMember(value, GetMultipleOfString())) - if (v->IsNumber() && v->GetDouble() > 0.0) - multipleOf_.CopyFrom(*v, *allocator_); - } - - ~Schema() { - AllocatorType::Free(enum_); - if (properties_) { - for (SizeType i = 0; i < propertyCount_; i++) - properties_[i].~Property(); - AllocatorType::Free(properties_); - } - if (patternProperties_) { - for (SizeType i = 0; i < patternPropertyCount_; i++) - patternProperties_[i].~PatternProperty(); - AllocatorType::Free(patternProperties_); - } - AllocatorType::Free(itemsTuple_); -#if RAPIDJSON_SCHEMA_HAS_REGEX - if (pattern_) { - pattern_->~RegexType(); - AllocatorType::Free(pattern_); - } -#endif - } - - bool BeginValue(Context& context) const { - if (context.inArray) { - if (uniqueItems_) - context.valueUniqueness = true; - - if (itemsList_) - context.valueSchema = itemsList_; - else if (itemsTuple_) { - if (context.arrayElementIndex < itemsTupleCount_) - context.valueSchema = itemsTuple_[context.arrayElementIndex]; - else if (additionalItemsSchema_) - context.valueSchema = additionalItemsSchema_; - else if (additionalItems_) - context.valueSchema = typeless_; - else - RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); - } - else - context.valueSchema = typeless_; - - context.arrayElementIndex++; - } - return true; - } - - RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { - if (context.patternPropertiesValidatorCount > 0) { - bool otherValid = false; - SizeType count = context.patternPropertiesValidatorCount; - if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) - otherValid = context.patternPropertiesValidators[--count]->IsValid(); - - bool patternValid = true; - for (SizeType i = 0; i < count; i++) - if (!context.patternPropertiesValidators[i]->IsValid()) { - patternValid = false; - break; - } - - if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { - if (!patternValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { - if (!patternValid || !otherValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - - if (enum_) { - const uint64_t h = context.factory.GetHashCode(context.hasher); - for (SizeType i = 0; i < enumCount_; i++) - if (enum_[i] == h) - goto foundEnum; - RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); - foundEnum:; - } - - if (allOf_.schemas) - for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) - if (!context.validators[i]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); - - if (anyOf_.schemas) { - for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) - if (context.validators[i]->IsValid()) - goto foundAny; - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); - foundAny:; - } - - if (oneOf_.schemas) { - bool oneValid = false; - for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) - if (context.validators[i]->IsValid()) { - if (oneValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - else - oneValid = true; - } - if (!oneValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - } - - if (not_ && context.validators[notValidatorIndex_]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); - - return true; - } - - bool Null(Context& context) const { - if (!(type_ & (1 << kNullSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - return CreateParallelValidator(context); - } - - bool Bool(Context& context, bool) const { - if (!(type_ & (1 << kBooleanSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - return CreateParallelValidator(context); - } - - bool Int(Context& context, int i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); - } - - bool Uint(Context& context, unsigned u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } - - bool Int64(Context& context, int64_t i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); - } - - bool Uint64(Context& context, uint64_t u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } - - bool Double(Context& context, double d) const { - if (!(type_ & (1 << kNumberSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) - return false; - - if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) - return false; - - if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) - return false; - - return CreateParallelValidator(context); - } - - bool String(Context& context, const Ch* str, SizeType length, bool) const { - if (!(type_ & (1 << kStringSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (minLength_ != 0 || maxLength_ != SizeType(~0)) { - SizeType count; - if (internal::CountStringCodePoint(str, length, &count)) { - if (count < minLength_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); - if (count > maxLength_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); - } - } - - if (pattern_ && !IsPatternMatch(pattern_, str, length)) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); - - return CreateParallelValidator(context); - } - - bool StartObject(Context& context) const { - if (!(type_ & (1 << kObjectSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (hasDependencies_ || hasRequired_) { - context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); - std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); - } - - if (patternProperties_) { // pre-allocate schema array - SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType - context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); - context.patternPropertiesSchemaCount = 0; - std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); - } - - return CreateParallelValidator(context); - } - - bool Key(Context& context, const Ch* str, SizeType len, bool) const { - if (patternProperties_) { - context.patternPropertiesSchemaCount = 0; - for (SizeType i = 0; i < patternPropertyCount_; i++) - if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; - context.valueSchema = typeless_; - } - } - - SizeType index; - if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { - if (context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; - context.valueSchema = typeless_; - context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; - } - else - context.valueSchema = properties_[index].schema; - - if (context.propertyExist) - context.propertyExist[index] = true; - - return true; - } - - if (additionalPropertiesSchema_) { - if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; - context.valueSchema = typeless_; - context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; - } - else - context.valueSchema = additionalPropertiesSchema_; - return true; - } - else if (additionalProperties_) { - context.valueSchema = typeless_; - return true; - } - - if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); - - return true; - } - - bool EndObject(Context& context, SizeType memberCount) const { - if (hasRequired_) - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].required) - if (!context.propertyExist[index]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); - - if (memberCount < minProperties_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); - - if (memberCount > maxProperties_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); - - if (hasDependencies_) { - for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) - if (context.propertyExist[sourceIndex]) { - if (properties_[sourceIndex].dependencies) { - for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) - if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - else if (properties_[sourceIndex].dependenciesSchema) - if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - } - - return true; - } - - bool StartArray(Context& context) const { - if (!(type_ & (1 << kArraySchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - context.arrayElementIndex = 0; - context.inArray = true; - - return CreateParallelValidator(context); - } - - bool EndArray(Context& context, SizeType elementCount) const { - context.inArray = false; - - if (elementCount < minItems_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); - - if (elementCount > maxItems_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); - - return true; - } - - // Generate functions for string literal according to Ch -#define RAPIDJSON_STRING_(name, ...) \ - static const ValueType& Get##name##String() {\ - static const Ch s[] = { __VA_ARGS__, '\0' };\ - static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ - return v;\ - } - - RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') - RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') - RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') - RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') - RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') - RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') - RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') - RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') - RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') - RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') - RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') - RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') - RAPIDJSON_STRING_(Not, 'n', 'o', 't') - RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') - RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') - RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') - RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') - -#undef RAPIDJSON_STRING_ - -private: - enum SchemaValueType { - kNullSchemaType, - kBooleanSchemaType, - kObjectSchemaType, - kArraySchemaType, - kStringSchemaType, - kNumberSchemaType, - kIntegerSchemaType, - kTotalSchemaType - }; - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - typedef internal::GenericRegex RegexType; -#elif RAPIDJSON_SCHEMA_USE_STDREGEX - typedef std::basic_regex RegexType; -#else - typedef char RegexType; -#endif - - struct SchemaArray { - SchemaArray() : schemas(), count() {} - ~SchemaArray() { AllocatorType::Free(schemas); } - const SchemaType** schemas; - SizeType begin; // begin index of context.validators - SizeType count; - }; - - template - void AddUniqueElement(V1& a, const V2& v) { - for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) - if (*itr == v) - return; - V1 c(v, *allocator_); - a.PushBack(c, *allocator_); - } - - static const ValueType* GetMember(const ValueType& value, const ValueType& name) { - typename ValueType::ConstMemberIterator itr = value.FindMember(name); - return itr != value.MemberEnd() ? &(itr->value) : 0; - } - - static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsBool()) - out = v->GetBool(); - } - - static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) - out = static_cast(v->GetUint64()); - } - - void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { - if (const ValueType* v = GetMember(value, name)) { - if (v->IsArray() && v->Size() > 0) { - PointerType q = p.Append(name, allocator_); - out.count = v->Size(); - out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); - memset(out.schemas, 0, sizeof(Schema*)* out.count); - for (SizeType i = 0; i < out.count; i++) - schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); - out.begin = validatorCount_; - validatorCount_ += out.count; - } - } - } - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - template - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) { - RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); - if (!r->IsValid()) { - r->~RegexType(); - AllocatorType::Free(r); - r = 0; - } - return r; - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { - GenericRegexSearch rs(*pattern); - return rs.Search(str); - } -#elif RAPIDJSON_SCHEMA_USE_STDREGEX - template - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) - try { - return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); - } - catch (const std::regex_error&) { - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { - std::match_results r; - return std::regex_search(str, str + length, r, *pattern); - } -#else - template - RegexType* CreatePattern(const ValueType&) { return 0; } - - static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } -#endif // RAPIDJSON_SCHEMA_USE_STDREGEX - - void AddType(const ValueType& type) { - if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; - else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; - else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; - else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; - else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; - else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; - else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); - } - - bool CreateParallelValidator(Context& context) const { - if (enum_ || context.arrayUniqueness) - context.hasher = context.factory.CreateHasher(); - - if (validatorCount_) { - RAPIDJSON_ASSERT(context.validators == 0); - context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); - context.validatorCount = validatorCount_; - - if (allOf_.schemas) - CreateSchemaValidators(context, allOf_); - - if (anyOf_.schemas) - CreateSchemaValidators(context, anyOf_); - - if (oneOf_.schemas) - CreateSchemaValidators(context, oneOf_); - - if (not_) - context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); - - if (hasSchemaDependencies_) { - for (SizeType i = 0; i < propertyCount_; i++) - if (properties_[i].dependenciesSchema) - context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); - } - } - - return true; - } - - void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { - for (SizeType i = 0; i < schemas.count; i++) - context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); - } - - // O(n) - bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { - SizeType len = name.GetStringLength(); - const Ch* str = name.GetString(); - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].name.GetStringLength() == len && - (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) - { - *outIndex = index; - return true; - } - return false; - } - - bool CheckInt(Context& context, int64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull()) { - if (minimum_.IsInt64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - else if (minimum_.IsUint64()) { - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() - } - else if (!CheckDoubleMinimum(context, static_cast(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsInt64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - else if (maximum_.IsUint64()) { } - /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() - else if (!CheckDoubleMaximum(context, static_cast(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - else if (!CheckDoubleMultipleOf(context, static_cast(i))) - return false; - } - - return true; - } - - bool CheckUint(Context& context, uint64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull()) { - if (minimum_.IsUint64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - else if (minimum_.IsInt64()) - /* do nothing */; // i >= 0 > minimum.Getint64() - else if (!CheckDoubleMinimum(context, static_cast(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsUint64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - else if (maximum_.IsInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ - else if (!CheckDoubleMaximum(context, static_cast(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (i % multipleOf_.GetUint64() != 0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - else if (!CheckDoubleMultipleOf(context, static_cast(i))) - return false; - } - - return true; - } - - bool CheckDoubleMinimum(Context& context, double d) const { - if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - return true; - } - - bool CheckDoubleMaximum(Context& context, double d) const { - if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - return true; - } - - bool CheckDoubleMultipleOf(Context& context, double d) const { - double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); - double q = std::floor(a / b); - double r = a - q * b; - if (r > 0.0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - return true; - } - - struct Property { - Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} - ~Property() { AllocatorType::Free(dependencies); } - SValue name; - const SchemaType* schema; - const SchemaType* dependenciesSchema; - SizeType dependenciesValidatorIndex; - bool* dependencies; - bool required; - }; - - struct PatternProperty { - PatternProperty() : schema(), pattern() {} - ~PatternProperty() { - if (pattern) { - pattern->~RegexType(); - AllocatorType::Free(pattern); - } - } - const SchemaType* schema; - RegexType* pattern; - }; - - AllocatorType* allocator_; - const SchemaType* typeless_; - uint64_t* enum_; - SizeType enumCount_; - SchemaArray allOf_; - SchemaArray anyOf_; - SchemaArray oneOf_; - const SchemaType* not_; - unsigned type_; // bitmask of kSchemaType - SizeType validatorCount_; - SizeType notValidatorIndex_; - - Property* properties_; - const SchemaType* additionalPropertiesSchema_; - PatternProperty* patternProperties_; - SizeType patternPropertyCount_; - SizeType propertyCount_; - SizeType minProperties_; - SizeType maxProperties_; - bool additionalProperties_; - bool hasDependencies_; - bool hasRequired_; - bool hasSchemaDependencies_; - - const SchemaType* additionalItemsSchema_; - const SchemaType* itemsList_; - const SchemaType** itemsTuple_; - SizeType itemsTupleCount_; - SizeType minItems_; - SizeType maxItems_; - bool additionalItems_; - bool uniqueItems_; - - RegexType* pattern_; - SizeType minLength_; - SizeType maxLength_; - - SValue minimum_; - SValue maximum_; - SValue multipleOf_; - bool exclusiveMinimum_; - bool exclusiveMaximum_; -}; - -template -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - *documentStack.template Push() = '/'; - char buffer[21]; - size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); - for (size_t i = 0; i < length; i++) - *documentStack.template Push() = static_cast(buffer[i]); - } -}; - -// Partial specialized version for char to prevent buffer copying. -template -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - if (sizeof(SizeType) == 4) { - char *buffer = documentStack.template Push(1 + 10); // '/' + uint - *buffer++ = '/'; - const char* end = internal::u32toa(index, buffer); - documentStack.template Pop(static_cast(10 - (end - buffer))); - } - else { - char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 - *buffer++ = '/'; - const char* end = internal::u64toa(index, buffer); - documentStack.template Pop(static_cast(20 - (end - buffer))); - } - } -}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// IGenericRemoteSchemaDocumentProvider - -template -class IGenericRemoteSchemaDocumentProvider { -public: - typedef typename SchemaDocumentType::Ch Ch; - - virtual ~IGenericRemoteSchemaDocumentProvider() {} - virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericSchemaDocument - -//! JSON schema document. -/*! - A JSON schema document is a compiled version of a JSON schema. - It is basically a tree of internal::Schema. - - \note This is an immutable class (i.e. its instance cannot be modified after construction). - \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. - \tparam Allocator Allocator type for allocating memory of this document. -*/ -template -class GenericSchemaDocument { -public: - typedef ValueT ValueType; - typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; - typedef Allocator AllocatorType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef internal::Schema SchemaType; - typedef GenericPointer PointerType; - friend class internal::Schema; - template - friend class GenericSchemaValidator; - - //! Constructor. - /*! - Compile a JSON document into schema document. - - \param document A JSON document as source. - \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. - \param allocator An optional allocator instance for allocating memory. Can be null. - */ - explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : - remoteProvider_(remoteProvider), - allocator_(allocator), - ownAllocator_(), - root_(), - typeless_(), - schemaMap_(allocator, kInitialSchemaMapSize), - schemaRef_(allocator, kInitialSchemaRefSize) - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - - typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); - new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); - - // Generate root schema, it will call CreateSchema() to create sub-schemas, - // And call AddRefSchema() if there are $ref. - CreateSchemaRecursive(&root_, PointerType(), document, document); - - // Resolve $ref - while (!schemaRef_.Empty()) { - SchemaRefEntry* refEntry = schemaRef_.template Pop(1); - if (const SchemaType* s = GetSchema(refEntry->target)) { - if (refEntry->schema) - *refEntry->schema = s; - - // Create entry in map if not exist - if (!GetSchema(refEntry->source)) { - new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); - } - } - else if (refEntry->schema) - *refEntry->schema = typeless_; - - refEntry->~SchemaRefEntry(); - } - - RAPIDJSON_ASSERT(root_ != 0); - - schemaRef_.ShrinkToFit(); // Deallocate all memory for ref - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : - remoteProvider_(rhs.remoteProvider_), - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - root_(rhs.root_), - typeless_(rhs.typeless_), - schemaMap_(std::move(rhs.schemaMap_)), - schemaRef_(std::move(rhs.schemaRef_)) - { - rhs.remoteProvider_ = 0; - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.typeless_ = 0; - } -#endif - - //! Destructor - ~GenericSchemaDocument() { - while (!schemaMap_.Empty()) - schemaMap_.template Pop(1)->~SchemaEntry(); - - if (typeless_) { - typeless_->~SchemaType(); - Allocator::Free(typeless_); - } - - RAPIDJSON_DELETE(ownAllocator_); - } - - //! Get the root schema. - const SchemaType& GetRoot() const { return *root_; } - -private: - //! Prohibit copying - GenericSchemaDocument(const GenericSchemaDocument&); - //! Prohibit assignment - GenericSchemaDocument& operator=(const GenericSchemaDocument&); - - struct SchemaRefEntry { - SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} - PointerType source; - PointerType target; - const SchemaType** schema; - }; - - struct SchemaEntry { - SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} - ~SchemaEntry() { - if (owned) { - schema->~SchemaType(); - Allocator::Free(schema); - } - } - PointerType pointer; - SchemaType* schema; - bool owned; - }; - - void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - if (schema) - *schema = typeless_; - - if (v.GetType() == kObjectType) { - const SchemaType* s = GetSchema(pointer); - if (!s) - CreateSchema(schema, pointer, v, document); - - for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) - CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); - } - else if (v.GetType() == kArrayType) - for (SizeType i = 0; i < v.Size(); i++) - CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); - } - - void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - RAPIDJSON_ASSERT(pointer.IsValid()); - if (v.IsObject()) { - if (!HandleRefSchema(pointer, schema, v, document)) { - SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); - new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); - if (schema) - *schema = s; - } - } - } - - bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { - static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; - static const ValueType kRefValue(kRefString, 4); - - typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); - if (itr == v.MemberEnd()) - return false; - - if (itr->value.IsString()) { - SizeType len = itr->value.GetStringLength(); - if (len > 0) { - const Ch* s = itr->value.GetString(); - SizeType i = 0; - while (i < len && s[i] != '#') // Find the first # - i++; - - if (i > 0) { // Remote reference, resolve immediately - if (remoteProvider_) { - if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { - if (schema) - *schema = sc; - return true; - } - } - } - } - } - else if (s[i] == '#') { // Local reference, defer resolution - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const ValueType* nv = pointer.Get(document)) - if (HandleRefSchema(source, schema, *nv, document)) - return true; - - new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); - return true; - } - } - } - } - return false; - } - - const SchemaType* GetSchema(const PointerType& pointer) const { - for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) - if (pointer == target->pointer) - return target->schema; - return 0; - } - - PointerType GetPointer(const SchemaType* schema) const { - for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) - if (schema == target->schema) - return target->pointer; - return PointerType(); - } - - const SchemaType* GetTypeless() const { return typeless_; } - - static const size_t kInitialSchemaMapSize = 64; - static const size_t kInitialSchemaRefSize = 64; - - IRemoteSchemaDocumentProviderType* remoteProvider_; - Allocator *allocator_; - Allocator *ownAllocator_; - const SchemaType* root_; //!< Root schema. - SchemaType* typeless_; - internal::Stack schemaMap_; // Stores created Pointer -> Schemas - internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref -}; - -//! GenericSchemaDocument using Value type. -typedef GenericSchemaDocument SchemaDocument; -//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. -typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; - -/////////////////////////////////////////////////////////////////////////////// -// GenericSchemaValidator - -//! JSON Schema Validator. -/*! - A SAX style JSON schema validator. - It uses a \c GenericSchemaDocument to validate SAX events. - It delegates the incoming SAX events to an output handler. - The default output handler does nothing. - It can be reused multiple times by calling \c Reset(). - - \tparam SchemaDocumentType Type of schema document. - \tparam OutputHandler Type of output handler. Default handler does nothing. - \tparam StateAllocator Allocator for storing the internal validation states. -*/ -template < - typename SchemaDocumentType, - typename OutputHandler = BaseReaderHandler, - typename StateAllocator = CrtAllocator> -class GenericSchemaValidator : - public internal::ISchemaStateFactory, - public internal::ISchemaValidator -{ -public: - typedef typename SchemaDocumentType::SchemaType SchemaType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename SchemaType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - - //! Constructor without output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - outputHandler_(0), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif - { - } - - //! Constructor with output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - OutputHandler& outputHandler, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - outputHandler_(&outputHandler), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif - { - } - - //! Destructor. - ~GenericSchemaValidator() { - Reset(); - RAPIDJSON_DELETE(ownStateAllocator_); - } - - //! Reset the internal states. - void Reset() { - while (!schemaStack_.Empty()) - PopSchema(); - documentStack_.Clear(); - valid_ = true; - } - - //! Checks whether the current state is valid. - // Implementation of ISchemaValidator - virtual bool IsValid() const { return valid_; } - - //! Gets the JSON pointer pointed to the invalid schema. - PointerType GetInvalidSchemaPointer() const { - return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); - } - - //! Gets the keyword of invalid schema. - const Ch* GetInvalidSchemaKeyword() const { - return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; - } - - //! Gets the JSON pointer pointed to the invalid value. - PointerType GetInvalidDocumentPointer() const { - return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); - } - -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - *documentStack_.template Push() = '\0';\ - documentStack_.template Pop(1);\ - internal::PrintInvalidDocument(documentStack_.template Bottom());\ -RAPIDJSON_MULTILINEMACRO_END -#else -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() -#endif - -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ - if (!valid_) return false; \ - if (!BeginValue() || !CurrentSchema().method arg1) {\ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ - return valid_ = false;\ - } - -#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ - for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ - if (context->hasher)\ - static_cast(context->hasher)->method arg2;\ - if (context->validators)\ - for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ - static_cast(context->validators[i_])->method arg2;\ - if (context->patternPropertiesValidators)\ - for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ - static_cast(context->patternPropertiesValidators[i_])->method arg2;\ - } - -#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ - return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) - -#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ - RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) - - bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } - bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } - bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } - bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } - bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } - bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } - bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } - bool RawNumber(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - bool String(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - - bool StartObject() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); - return valid_ = !outputHandler_ || outputHandler_->StartObject(); - } - - bool Key(const Ch* str, SizeType len, bool copy) { - if (!valid_) return false; - AppendToken(str, len); - if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); - return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); - } - - bool EndObject(SizeType memberCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); - if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); - } - - bool StartArray() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); - return valid_ = !outputHandler_ || outputHandler_->StartArray(); - } - - bool EndArray(SizeType elementCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); - if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); - } - -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ -#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ -#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ - - // Implementation of ISchemaStateFactory - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { - return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, -#if RAPIDJSON_SCHEMA_VERBOSE - depth_ + 1, -#endif - &GetStateAllocator()); - } - - virtual void DestroySchemaValidator(ISchemaValidator* validator) { - GenericSchemaValidator* v = static_cast(validator); - v->~GenericSchemaValidator(); - StateAllocator::Free(v); - } - - virtual void* CreateHasher() { - return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); - } - - virtual uint64_t GetHashCode(void* hasher) { - return static_cast(hasher)->GetHashCode(); - } - - virtual void DestroryHasher(void* hasher) { - HasherType* h = static_cast(hasher); - h->~HasherType(); - StateAllocator::Free(h); - } - - virtual void* MallocState(size_t size) { - return GetStateAllocator().Malloc(size); - } - - virtual void FreeState(void* p) { - StateAllocator::Free(p); - } - -private: - typedef typename SchemaType::Context Context; - typedef GenericValue, StateAllocator> HashCodeArray; - typedef internal::Hasher HasherType; - - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - const SchemaType& root, -#if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth, -#endif - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(root), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - outputHandler_(0), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(depth) -#endif - { - } - - StateAllocator& GetStateAllocator() { - if (!stateAllocator_) - stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator)(); - return *stateAllocator_; - } - - bool BeginValue() { - if (schemaStack_.Empty()) - PushSchema(root_); - else { - if (CurrentContext().inArray) - internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); - - if (!CurrentSchema().BeginValue(CurrentContext())) - return false; - - SizeType count = CurrentContext().patternPropertiesSchemaCount; - const SchemaType** sa = CurrentContext().patternPropertiesSchemas; - typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; - bool valueUniqueness = CurrentContext().valueUniqueness; - RAPIDJSON_ASSERT(CurrentContext().valueSchema); - PushSchema(*CurrentContext().valueSchema); - - if (count > 0) { - CurrentContext().objectPatternValidatorType = patternValidatorType; - ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; - SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; - va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); - for (SizeType i = 0; i < count; i++) - va[validatorCount++] = CreateSchemaValidator(*sa[i]); - } - - CurrentContext().arrayUniqueness = valueUniqueness; - } - return true; - } - - bool EndValue() { - if (!CurrentSchema().EndValue(CurrentContext())) - return false; - -#if RAPIDJSON_SCHEMA_VERBOSE - GenericStringBuffer sb; - schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); - - *documentStack_.template Push() = '\0'; - documentStack_.template Pop(1); - internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); -#endif - - uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; - - PopSchema(); - - if (!schemaStack_.Empty()) { - Context& context = CurrentContext(); - if (context.valueUniqueness) { - HashCodeArray* a = static_cast(context.arrayElementHashCodes); - if (!a) - CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); - for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) - if (itr->GetUint64() == h) - RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); - a->PushBack(h, GetStateAllocator()); - } - } - - // Remove the last token of document pointer - while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') - ; - - return true; - } - - void AppendToken(const Ch* str, SizeType len) { - documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters - *documentStack_.template PushUnsafe() = '/'; - for (SizeType i = 0; i < len; i++) { - if (str[i] == '~') { - *documentStack_.template PushUnsafe() = '~'; - *documentStack_.template PushUnsafe() = '0'; - } - else if (str[i] == '/') { - *documentStack_.template PushUnsafe() = '~'; - *documentStack_.template PushUnsafe() = '1'; - } - else - *documentStack_.template PushUnsafe() = str[i]; - } - } - - RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } - - RAPIDJSON_FORCEINLINE void PopSchema() { - Context* c = schemaStack_.template Pop(1); - if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { - a->~HashCodeArray(); - StateAllocator::Free(a); - } - c->~Context(); - } - - const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } - Context& CurrentContext() { return *schemaStack_.template Top(); } - const Context& CurrentContext() const { return *schemaStack_.template Top(); } - - static const size_t kDefaultSchemaStackCapacity = 1024; - static const size_t kDefaultDocumentStackCapacity = 256; - const SchemaDocumentType* schemaDocument_; - const SchemaType& root_; - StateAllocator* stateAllocator_; - StateAllocator* ownStateAllocator_; - internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) - internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) - OutputHandler* outputHandler_; - bool valid_; -#if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth_; -#endif -}; - -typedef GenericSchemaValidator SchemaValidator; - -/////////////////////////////////////////////////////////////////////////////// -// SchemaValidatingReader - -//! A helper class for parsing with validation. -/*! - This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). - - \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept. - \tparam SourceEncoding Encoding of the input stream. - \tparam SchemaDocumentType Type of schema document. - \tparam StackAllocator Allocator type for stack. -*/ -template < - unsigned parseFlags, - typename InputStream, - typename SourceEncoding, - typename SchemaDocumentType = SchemaDocument, - typename StackAllocator = CrtAllocator> -class SchemaValidatingReader { -public: - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename InputStream::Ch Ch; - - //! Constructor - /*! - \param is Input stream. - \param sd Schema document. - */ - SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} - - template - bool operator()(Handler& handler) { - GenericReader reader; - GenericSchemaValidator validator(sd_, handler); - parseResult_ = reader.template Parse(is_, validator); - - isValid_ = validator.IsValid(); - if (isValid_) { - invalidSchemaPointer_ = PointerType(); - invalidSchemaKeyword_ = 0; - invalidDocumentPointer_ = PointerType(); - } - else { - invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); - invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); - invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); - } - - return parseResult_; - } - - const ParseResult& GetParseResult() const { return parseResult_; } - bool IsValid() const { return isValid_; } - const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } - const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } - const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } - -private: - InputStream& is_; - const SchemaDocumentType& sd_; - - ParseResult parseResult_; - PointerType invalidSchemaPointer_; - const Ch* invalidSchemaKeyword_; - PointerType invalidDocumentPointer_; - bool isValid_; -}; - -RAPIDJSON_NAMESPACE_END -RAPIDJSON_DIAG_POP - -#endif // RAPIDJSON_SCHEMA_H_ diff --git a/contrib/rapidjson/include/rapidjson/stream.h b/contrib/rapidjson/include/rapidjson/stream.h deleted file mode 100644 index fef82c252..000000000 --- a/contrib/rapidjson/include/rapidjson/stream.h +++ /dev/null @@ -1,179 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#include "rapidjson.h" - -#ifndef RAPIDJSON_STREAM_H_ -#define RAPIDJSON_STREAM_H_ - -#include "encodings.h" - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Stream - -/*! \class rapidjson::Stream - \brief Concept for reading and writing characters. - - For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). - - For write-only stream, only need to implement Put() and Flush(). - -\code -concept Stream { - typename Ch; //!< Character type of the stream. - - //! Read the current character from stream without moving the read cursor. - Ch Peek() const; - - //! Read the current character from stream and moving the read cursor to next character. - Ch Take(); - - //! Get the current read cursor. - //! \return Number of characters read from start. - size_t Tell(); - - //! Begin writing operation at the current read pointer. - //! \return The begin writer pointer. - Ch* PutBegin(); - - //! Write a character. - void Put(Ch c); - - //! Flush the buffer. - void Flush(); - - //! End the writing operation. - //! \param begin The begin write pointer returned by PutBegin(). - //! \return Number of characters written. - size_t PutEnd(Ch* begin); -} -\endcode -*/ - -//! Provides additional information for stream. -/*! - By using traits pattern, this type provides a default configuration for stream. - For custom stream, this type can be specialized for other configuration. - See TEST(Reader, CustomStringStream) in readertest.cpp for example. -*/ -template -struct StreamTraits { - //! Whether to make local copy of stream for optimization during parsing. - /*! - By default, for safety, streams do not use local copy optimization. - Stream that can be copied fast should specialize this, like StreamTraits. - */ - enum { copyOptimization = 0 }; -}; - -//! Reserve n characters for writing to a stream. -template -inline void PutReserve(Stream& stream, size_t count) { - (void)stream; - (void)count; -} - -//! Write character to a stream, presuming buffer is reserved. -template -inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { - stream.Put(c); -} - -//! Put N copies of a character to a stream. -template -inline void PutN(Stream& stream, Ch c, size_t n) { - PutReserve(stream, n); - for (size_t i = 0; i < n; i++) - PutUnsafe(stream, c); -} - -/////////////////////////////////////////////////////////////////////////////// -// StringStream - -//! Read-only string stream. -/*! \note implements Stream concept -*/ -template -struct GenericStringStream { - typedef typename Encoding::Ch Ch; - - GenericStringStream(const Ch *src) : src_(src), head_(src) {} - - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast(src_ - head_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! String stream with UTF8 encoding. -typedef GenericStringStream > StringStream; - -/////////////////////////////////////////////////////////////////////////////// -// InsituStringStream - -//! A read-write string stream. -/*! This string stream is particularly designed for in-situ parsing. - \note implements Stream concept -*/ -template -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; - - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return static_cast(src_ - head_); } - - // Write - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - - Ch* PutBegin() { return dst_ = src_; } - size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } - void Flush() {} - - Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } - void Pop(size_t count) { dst_ -= count; } - - Ch* src_; - Ch* dst_; - Ch* head_; -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! Insitu string stream with UTF8 encoding. -typedef GenericInsituStringStream > InsituStringStream; - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_STREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/stringbuffer.h b/contrib/rapidjson/include/rapidjson/stringbuffer.h index 4e38b82c3..1c9c80b79 100644 --- a/contrib/rapidjson/include/rapidjson/stringbuffer.h +++ b/contrib/rapidjson/include/rapidjson/stringbuffer.h @@ -15,8 +15,7 @@ #ifndef RAPIDJSON_STRINGBUFFER_H_ #define RAPIDJSON_STRINGBUFFER_H_ -#include "stream.h" -#include "internal/stack.h" +#include "rapidjson.h" #if RAPIDJSON_HAS_CXX11_RVALUE_REFS #include // std::move @@ -24,11 +23,6 @@ #include "internal/stack.h" -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - RAPIDJSON_NAMESPACE_BEGIN //! Represents an in-memory output stream. @@ -54,7 +48,6 @@ public: #endif void Put(Ch c) { *stack_.template Push() = c; } - void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } void Flush() {} void Clear() { stack_.Clear(); } @@ -64,10 +57,7 @@ public: stack_.ShrinkToFit(); stack_.template Pop(1); } - - void Reserve(size_t count) { stack_.template Reserve(count); } Ch* Push(size_t count) { return stack_.template Push(count); } - Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } void Pop(size_t count) { stack_.template Pop(count); } const Ch* GetString() const { @@ -78,12 +68,8 @@ public: return stack_.template Bottom(); } - //! Get the size of string in bytes in the string buffer. size_t GetSize() const { return stack_.GetSize(); } - //! Get the length of string in Ch in the string buffer. - size_t GetLength() const { return stack_.GetSize() / sizeof(Ch); } - static const size_t kDefaultCapacity = 256; mutable internal::Stack stack_; @@ -96,16 +82,6 @@ private: //! String buffer with UTF8 encoding typedef GenericStringBuffer > StringBuffer; -template -inline void PutReserve(GenericStringBuffer& stream, size_t count) { - stream.Reserve(count); -} - -template -inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { - stream.PutUnsafe(c); -} - //! Implement specialized version of PutN() with memset() for better performance. template<> inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { @@ -114,8 +90,4 @@ inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { RAPIDJSON_NAMESPACE_END -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/contrib/rapidjson/include/rapidjson/writer.h b/contrib/rapidjson/include/rapidjson/writer.h index e610ebb60..e1eea38b9 100644 --- a/contrib/rapidjson/include/rapidjson/writer.h +++ b/contrib/rapidjson/include/rapidjson/writer.h @@ -15,8 +15,7 @@ #ifndef RAPIDJSON_WRITER_H_ #define RAPIDJSON_WRITER_H_ -#include "stream.h" -#include "internal/meta.h" +#include "rapidjson.h" #include "internal/stack.h" #include "internal/strfunc.h" #include "internal/dtoa.h" @@ -24,16 +23,8 @@ #include "stringbuffer.h" #include // placement new -#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) -#include -#pragma intrinsic(_BitScanForward) -#endif -#ifdef RAPIDJSON_SSE42 -#include -#elif defined(RAPIDJSON_SSE2) -#include -#elif defined(RAPIDJSON_NEON) -#include +#if RAPIDJSON_HAS_STDSTRING +#include #endif #ifdef _MSC_VER @@ -41,36 +32,8 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant #endif -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(unreachable-code) -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - RAPIDJSON_NAMESPACE_BEGIN -/////////////////////////////////////////////////////////////////////////////// -// WriteFlag - -/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS - \ingroup RAPIDJSON_CONFIG - \brief User-defined kWriteDefaultFlags definition. - - User can define this as any \c WriteFlag combinations. -*/ -#ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS -#define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags -#endif - -//! Combination of writeFlags -enum WriteFlag { - kWriteNoFlags = 0, //!< No flags are set. - kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. - kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. - kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS -}; - //! JSON writer /*! Writer implements the concept Handler. It generates JSON text by events to an output os. @@ -87,13 +50,11 @@ enum WriteFlag { \tparam StackAllocator Type of allocator for allocating memory of stack. \note implements Handler concept */ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> class Writer { public: typedef typename SourceEncoding::Ch Ch; - static const int kDefaultMaxDecimalPlaces = 324; - //! Constructor /*! \param os Output stream. \param stackAllocator User supplied allocator. If it is null, it will create a private one. @@ -101,18 +62,11 @@ public: */ explicit Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), hasRoot_(false) {} explicit Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Writer(Writer&& rhs) : - os_(rhs.os_), level_stack_(std::move(rhs.level_stack_)), maxDecimalPlaces_(rhs.maxDecimalPlaces_), hasRoot_(rhs.hasRoot_) { - rhs.os_ = 0; - } -#endif + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), hasRoot_(false) {} //! Reset the writer with a new stream. /*! @@ -146,66 +100,29 @@ public: return hasRoot_ && level_stack_.Empty(); } - int GetMaxDecimalPlaces() const { - return maxDecimalPlaces_; - } - - //! Sets the maximum number of decimal places for double output. - /*! - This setting truncates the output with specified number of decimal places. - - For example, - - \code - writer.SetMaxDecimalPlaces(3); - writer.StartArray(); - writer.Double(0.12345); // "0.123" - writer.Double(0.0001); // "0.0" - writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) - writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) - writer.EndArray(); - \endcode - - The default setting does not truncate any decimal places. You can restore to this setting by calling - \code - writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); - \endcode - */ - void SetMaxDecimalPlaces(int maxDecimalPlaces) { - maxDecimalPlaces_ = maxDecimalPlaces; - } - /*!@name Implementation of Handler \see Handler */ //@{ - bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } - bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } - bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } - bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } - bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } - bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } + bool Null() { Prefix(kNullType); return WriteNull(); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } + bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } + bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } + bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } //! Writes the given \c double value to the stream /*! \param d The value to be written. \return Whether it is succeed. */ - bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } - - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - RAPIDJSON_ASSERT(str != 0); - (void)copy; - Prefix(kNumberType); - return EndValue(WriteString(str, length)); - } + bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } bool String(const Ch* str, SizeType length, bool copy = false) { - RAPIDJSON_ASSERT(str != 0); (void)copy; Prefix(kStringType); - return EndValue(WriteString(str, length)); + return WriteString(str, length); } #if RAPIDJSON_HAS_STDSTRING @@ -221,21 +138,16 @@ public: } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - -#if RAPIDJSON_HAS_STDSTRING - bool Key(const std::basic_string& str) - { - return Key(str.data(), SizeType(str.size())); - } -#endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); // currently inside an Array, not Object - RAPIDJSON_ASSERT(0 == level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); level_stack_.template Pop(1); - return EndValue(WriteEndObject()); + bool ret = WriteEndObject(); + if (level_stack_.Empty()) // end of json text + os_->Flush(); + return ret; } bool StartArray() { @@ -249,7 +161,10 @@ public: RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); level_stack_.template Pop(1); - return EndValue(WriteEndArray()); + bool ret = WriteEndArray(); + if (level_stack_.Empty()) // end of json text + os_->Flush(); + return ret; } //@} @@ -257,33 +172,11 @@ public: //@{ //! Simpler but slower overload. - bool String(const Ch* const& str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* const& str) { return Key(str, internal::StrLen(str)); } - + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + //@} - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. - - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - */ - bool RawValue(const Ch* json, size_t length, Type type) { - RAPIDJSON_ASSERT(json != 0); - Prefix(type); - return EndValue(WriteRawValue(json, length)); - } - - //! Flush the output stream. - /*! - Allows the user to flush the output stream immediately. - */ - void Flush() { - os_->Flush(); - } - protected: //! Information for each nested level struct Level { @@ -295,18 +188,15 @@ protected: static const size_t kDefaultLevelDepth = 32; bool WriteNull() { - PutReserve(*os_, 4); - PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; + os_->Put('n'); os_->Put('u'); os_->Put('l'); os_->Put('l'); return true; } bool WriteBool(bool b) { if (b) { - PutReserve(*os_, 4); - PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); + os_->Put('t'); os_->Put('r'); os_->Put('u'); os_->Put('e'); } else { - PutReserve(*os_, 5); - PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); + os_->Put('f'); os_->Put('a'); os_->Put('l'); os_->Put('s'); os_->Put('e'); } return true; } @@ -314,69 +204,45 @@ protected: bool WriteInt(int i) { char buffer[11]; const char* end = internal::i32toa(i, buffer); - PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + os_->Put(*p); return true; } bool WriteUint(unsigned u) { char buffer[10]; const char* end = internal::u32toa(u, buffer); - PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + os_->Put(*p); return true; } bool WriteInt64(int64_t i64) { char buffer[21]; const char* end = internal::i64toa(i64, buffer); - PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + os_->Put(*p); return true; } bool WriteUint64(uint64_t u64) { char buffer[20]; char* end = internal::u64toa(u64, buffer); - PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + os_->Put(*p); return true; } bool WriteDouble(double d) { - if (internal::Double(d).IsNanOrInf()) { - if (!(writeFlags & kWriteNanAndInfFlag)) - return false; - if (internal::Double(d).IsNan()) { - PutReserve(*os_, 3); - PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); - return true; - } - if (internal::Double(d).Sign()) { - PutReserve(*os_, 9); - PutUnsafe(*os_, '-'); - } - else - PutReserve(*os_, 8); - PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); - PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); - return true; - } - char buffer[25]; - char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); - PutReserve(*os_, static_cast(end - buffer)); + char* end = internal::dtoa(d, buffer); for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + os_->Put(*p); return true; } bool WriteString(const Ch* str, SizeType length) { - static const typename OutputStream::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; static const char escape[256] = { #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 //0 1 2 3 4 5 6 7 8 9 A B C D E F @@ -389,27 +255,22 @@ protected: #undef Z16 }; - if (TargetEncoding::supportUnicode) - PutReserve(*os_, 2 + length * 6); // "\uxxxx..." - else - PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." - - PutUnsafe(*os_, '\"'); + os_->Put('\"'); GenericStringStream is(str); - while (ScanWriteUnescapedString(is, length)) { + while (is.Tell() < length) { const Ch c = is.Peek(); - if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { + if (!TargetEncoding::supportUnicode && (unsigned)c >= 0x80) { // Unicode escaping unsigned codepoint; - if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) + if (!SourceEncoding::Decode(is, &codepoint)) return false; - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, 'u'); + os_->Put('\\'); + os_->Put('u'); if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { - PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); + os_->Put(hexDigits[(codepoint >> 12) & 15]); + os_->Put(hexDigits[(codepoint >> 8) & 15]); + os_->Put(hexDigits[(codepoint >> 4) & 15]); + os_->Put(hexDigits[(codepoint ) & 15]); } else { RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); @@ -417,59 +278,45 @@ protected: unsigned s = codepoint - 0x010000; unsigned lead = (s >> 10) + 0xD800; unsigned trail = (s & 0x3FF) + 0xDC00; - PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(lead ) & 15]); - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, 'u'); - PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(trail ) & 15]); + os_->Put(hexDigits[(lead >> 12) & 15]); + os_->Put(hexDigits[(lead >> 8) & 15]); + os_->Put(hexDigits[(lead >> 4) & 15]); + os_->Put(hexDigits[(lead ) & 15]); + os_->Put('\\'); + os_->Put('u'); + os_->Put(hexDigits[(trail >> 12) & 15]); + os_->Put(hexDigits[(trail >> 8) & 15]); + os_->Put(hexDigits[(trail >> 4) & 15]); + os_->Put(hexDigits[(trail ) & 15]); } } - else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { + else if ((sizeof(Ch) == 1 || (unsigned)c < 256) && escape[(unsigned char)c]) { is.Take(); - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, static_cast(escape[static_cast(c)])); - if (escape[static_cast(c)] == 'u') { - PutUnsafe(*os_, '0'); - PutUnsafe(*os_, '0'); - PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); - PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); + os_->Put('\\'); + os_->Put(escape[(unsigned char)c]); + if (escape[(unsigned char)c] == 'u') { + os_->Put('0'); + os_->Put('0'); + os_->Put(hexDigits[(unsigned char)c >> 4]); + os_->Put(hexDigits[(unsigned char)c & 0xF]); } } - else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? - Transcoder::Validate(is, *os_) : - Transcoder::TranscodeUnsafe(is, *os_)))) - return false; + else + if (!Transcoder::Transcode(is, *os_)) + return false; } - PutUnsafe(*os_, '\"'); + os_->Put('\"'); return true; } - bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { - return RAPIDJSON_LIKELY(is.Tell() < length); - } - bool WriteStartObject() { os_->Put('{'); return true; } bool WriteEndObject() { os_->Put('}'); return true; } bool WriteStartArray() { os_->Put('['); return true; } bool WriteEndArray() { os_->Put(']'); return true; } - bool WriteRawValue(const Ch* json, size_t length) { - PutReserve(*os_, length); - for (size_t i = 0; i < length; i++) { - RAPIDJSON_ASSERT(json[i] != '\0'); - PutUnsafe(*os_, json[i]); - } - return true; - } - void Prefix(Type type) { (void)type; - if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root + if (level_stack_.GetSize() != 0) { // this value is not at root Level* level = level_stack_.template Top(); if (level->valueCount > 0) { if (level->inArray) @@ -487,16 +334,8 @@ protected: } } - // Flush the value if it is the top level one. - bool EndValue(bool ret) { - if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text - Flush(); - return ret; - } - OutputStream* os_; internal::Stack level_stack_; - int maxDecimalPlaces_; bool hasRoot_; private: @@ -511,7 +350,7 @@ template<> inline bool Writer::WriteInt(int i) { char *buffer = os_->Push(11); const char* end = internal::i32toa(i, buffer); - os_->Pop(static_cast(11 - (end - buffer))); + os_->Pop(11 - (end - buffer)); return true; } @@ -519,7 +358,7 @@ template<> inline bool Writer::WriteUint(unsigned u) { char *buffer = os_->Push(10); const char* end = internal::u32toa(u, buffer); - os_->Pop(static_cast(10 - (end - buffer))); + os_->Pop(10 - (end - buffer)); return true; } @@ -527,7 +366,7 @@ template<> inline bool Writer::WriteInt64(int64_t i64) { char *buffer = os_->Push(21); const char* end = internal::i64toa(i64, buffer); - os_->Pop(static_cast(21 - (end - buffer))); + os_->Pop(21 - (end - buffer)); return true; } @@ -535,177 +374,22 @@ template<> inline bool Writer::WriteUint64(uint64_t u) { char *buffer = os_->Push(20); const char* end = internal::u64toa(u, buffer); - os_->Pop(static_cast(20 - (end - buffer))); + os_->Pop(20 - (end - buffer)); return true; } template<> inline bool Writer::WriteDouble(double d) { - if (internal::Double(d).IsNanOrInf()) { - // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). - if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) - return false; - if (internal::Double(d).IsNan()) { - PutReserve(*os_, 3); - PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); - return true; - } - if (internal::Double(d).Sign()) { - PutReserve(*os_, 9); - PutUnsafe(*os_, '-'); - } - else - PutReserve(*os_, 8); - PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); - PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); - return true; - } - char *buffer = os_->Push(25); - char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); - os_->Pop(static_cast(25 - (end - buffer))); + char* end = internal::dtoa(d, buffer); + os_->Pop(25 - (end - buffer)); return true; } -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) -template<> -inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { - if (length < 16) - return RAPIDJSON_LIKELY(is.Tell() < length); - - if (!RAPIDJSON_LIKELY(is.Tell() < length)) - return false; - - const char* p = is.src_; - const char* end = is.head_ + length; - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); - if (nextAligned > end) - return true; - - while (p != nextAligned) - if (*p < 0x20 || *p == '\"' || *p == '\\') { - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); - } - else - os_->PutUnsafe(*p++); - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (; p != endAligned; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - SizeType len; -#ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - len = offset; -#else - len = static_cast(__builtin_ffs(r) - 1); -#endif - char* q = reinterpret_cast(os_->PushUnsafe(len)); - for (size_t i = 0; i < len; i++) - q[i] = p[i]; - - p += len; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); - } - - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); -} -#elif defined(RAPIDJSON_NEON) -template<> -inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { - if (length < 16) - return RAPIDJSON_LIKELY(is.Tell() < length); - - if (!RAPIDJSON_LIKELY(is.Tell() < length)) - return false; - - const char* p = is.src_; - const char* end = is.head_ + length; - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); - if (nextAligned > end) - return true; - - while (p != nextAligned) - if (*p < 0x20 || *p == '\"' || *p == '\\') { - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); - } - else - os_->PutUnsafe(*p++); - - // The rest of string using SIMD - const uint8x16_t s0 = vmovq_n_u8('"'); - const uint8x16_t s1 = vmovq_n_u8('\\'); - const uint8x16_t s2 = vmovq_n_u8('\b'); - const uint8x16_t s3 = vmovq_n_u8(32); - - for (; p != endAligned; p += 16) { - const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); - uint8x16_t x = vceqq_u8(s, s0); - x = vorrq_u8(x, vceqq_u8(s, s1)); - x = vorrq_u8(x, vceqq_u8(s, s2)); - x = vorrq_u8(x, vcltq_u8(s, s3)); - - x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract - - SizeType len = 0; - bool escaped = false; - if (low == 0) { - if (high != 0) { - unsigned lz = (unsigned)__builtin_clzll(high); - len = 8 + (lz >> 3); - escaped = true; - } - } else { - unsigned lz = (unsigned)__builtin_clzll(low); - len = lz >> 3; - escaped = true; - } - if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped - char* q = reinterpret_cast(os_->PushUnsafe(len)); - for (size_t i = 0; i < len; i++) - q[i] = p[i]; - - p += len; - break; - } - vst1q_u8(reinterpret_cast(os_->PushUnsafe(16)), s); - } - - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); -} -#endif // RAPIDJSON_NEON - RAPIDJSON_NAMESPACE_END #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/contrib/rapidjson/license.txt b/contrib/rapidjson/license.txt index 7ccc161c8..879293afa 100644 --- a/contrib/rapidjson/license.txt +++ b/contrib/rapidjson/license.txt @@ -3,7 +3,7 @@ Tencent is pleased to support the open source community by making RapidJSON avai Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. +If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. A copy of the MIT License is included in this file. Other dependencies and licenses: diff --git a/contrib/rapidjson/readme.md b/contrib/rapidjson/readme.md index b833a98e8..19da38667 100644 --- a/contrib/rapidjson/readme.md +++ b/contrib/rapidjson/readme.md @@ -1,17 +1,17 @@ -![RapidJSON logo](doc/logo/rapidjson.png) +![](doc/logo/rapidjson.png) -![Release version](https://img.shields.io/badge/release-v1.1.0-blue.svg) +![](https://img.shields.io/badge/release-v1.0.2-blue.png) -## A fast JSON parser/generator for C++ with both SAX/DOM style API +## A fast JSON parser/generator for C++ with both SAX/DOM style API Tencent is pleased to support the open source community by making RapidJSON available. Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -* [RapidJSON GitHub](https://github.com/Tencent/rapidjson/) +* [RapidJSON GitHub](https://github.com/miloyip/rapidjson/) * RapidJSON Documentation - * [English](http://rapidjson.org/) - * [简体中文](http://rapidjson.org/zh-cn/) + * [English](http://miloyip.github.io/rapidjson/) + * [简体中文](http://miloyip.github.io/rapidjson/zh-cn/) * [GitBook](https://www.gitbook.com/book/miloyip/rapidjson/) with downloadable PDF/EPUB/MOBI, without API reference. ## Build status @@ -20,43 +20,33 @@ Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights | :---------------: | :-----------------: | :-------------------: | | ![lin-badge] | ![win-badge] | ![cov-badge] | -[lin-badge]: https://travis-ci.org/Tencent/rapidjson.svg?branch=master "Travis build status" -[lin-link]: https://travis-ci.org/Tencent/rapidjson "Travis build status" -[win-badge]: https://ci.appveyor.com/api/projects/status/l6qulgqahcayidrf/branch/master?svg=true "AppVeyor build status" -[win-link]: https://ci.appveyor.com/project/miloyip/rapidjson-0fdqj/branch/master "AppVeyor build status" -[cov-badge]: https://coveralls.io/repos/Tencent/rapidjson/badge.svg?branch=master "Coveralls coverage" -[cov-link]: https://coveralls.io/r/Tencent/rapidjson?branch=master "Coveralls coverage" +[lin-badge]: https://travis-ci.org/miloyip/rapidjson.png?branch=master "Travis build status" +[lin-link]: https://travis-ci.org/miloyip/rapidjson "Travis build status" +[win-badge]: https://ci.appveyor.com/api/projects/status/u658dcuwxo14a8m9/branch/master "AppVeyor build status" +[win-link]: https://ci.appveyor.com/project/miloyip/rapidjson/branch/master "AppVeyor build status" +[cov-badge]: https://coveralls.io/repos/miloyip/rapidjson/badge.png?branch=master +[cov-link]: https://coveralls.io/r/miloyip/rapidjson?branch=master ## Introduction RapidJSON is a JSON parser and generator for C++. It was inspired by [RapidXml](http://rapidxml.sourceforge.net/). -* RapidJSON is **small** but **complete**. It supports both SAX and DOM style API. The SAX parser is only a half thousand lines of code. +* RapidJSON is small but complete. It supports both SAX and DOM style API. The SAX parser is only a half thousand lines of code. -* RapidJSON is **fast**. Its performance can be comparable to `strlen()`. It also optionally supports SSE2/SSE4.2 for acceleration. +* RapidJSON is fast. Its performance can be comparable to `strlen()`. It also optionally supports SSE2/SSE4.2 for acceleration. -* RapidJSON is **self-contained** and **header-only**. It does not depend on external libraries such as BOOST. It even does not depend on STL. +* RapidJSON is self-contained. It does not depend on external libraries such as BOOST. It even does not depend on STL. -* RapidJSON is **memory-friendly**. Each JSON value occupies exactly 16 bytes for most 32/64-bit machines (excluding text string). By default it uses a fast memory allocator, and the parser allocates memory compactly during parsing. +* RapidJSON is memory friendly. Each JSON value occupies exactly 16/20 bytes for most 32/64-bit machines (excluding text string). By default it uses a fast memory allocator, and the parser allocates memory compactly during parsing. -* RapidJSON is **Unicode-friendly**. It supports UTF-8, UTF-16, UTF-32 (LE & BE), and their detection, validation and transcoding internally. For example, you can read a UTF-8 file and let RapidJSON transcode the JSON strings into UTF-16 in the DOM. It also supports surrogates and "\u0000" (null character). +* RapidJSON is Unicode friendly. It supports UTF-8, UTF-16, UTF-32 (LE & BE), and their detection, validation and transcoding internally. For example, you can read a UTF-8 file and let RapidJSON transcode the JSON strings into UTF-16 in the DOM. It also supports surrogates and "\u0000" (null character). More features can be read [here](doc/features.md). -JSON(JavaScript Object Notation) is a light-weight data exchange format. RapidJSON should be in fully compliance with RFC7159/ECMA-404, with optional support of relaxed syntax. More information about JSON can be obtained at +JSON(JavaScript Object Notation) is a light-weight data exchange format. RapidJSON should be in fully compliance with RFC7159/ECMA-404. More information about JSON can be obtained at * [Introducing JSON](http://json.org/) -* [RFC7159: The JavaScript Object Notation (JSON) Data Interchange Format](https://tools.ietf.org/html/rfc7159) -* [Standard ECMA-404: The JSON Data Interchange Format](https://www.ecma-international.org/publications/standards/Ecma-404.htm) - -## Highlights in v1.1 (2016-8-25) - -* Added [JSON Pointer](doc/pointer.md) -* Added [JSON Schema](doc/schema.md) -* Added [relaxed JSON syntax](doc/dom.md) (comment, trailing comma, NaN/Infinity) -* Iterating array/object with [C++11 Range-based for loop](doc/tutorial.md) -* Reduce memory overhead of each `Value` from 24 bytes to 16 bytes in x86-64 architecture. - -For other changes please refer to [change log](CHANGELOG.md). +* [RFC7159: The JavaScript Object Notation (JSON) Data Interchange Format](http://www.ietf.org/rfc/rfc7159.txt) +* [Standard ECMA-404: The JSON Data Interchange Format](http://www.ecma-international.org/publications/standards/Ecma-404.htm) ## Compatibility @@ -73,9 +63,9 @@ Users can build and run the unit tests on their platform/compiler. RapidJSON is a header-only C++ library. Just copy the `include/rapidjson` folder to system or project's include path. RapidJSON uses following software as its dependencies: -* [CMake](https://cmake.org/) as a general build tool -* (optional) [Doxygen](http://www.doxygen.org) to build documentation -* (optional) [googletest](https://github.com/google/googletest) for unit and performance testing +* [CMake](http://www.cmake.org) as a general build tool +* (optional)[Doxygen](http://www.doxygen.org) to build documentation +* (optional)[googletest](https://code.google.com/p/googletest/) for unit and performance testing To generate user documentation and run tests please proceed with the steps below: @@ -84,7 +74,7 @@ To generate user documentation and run tests please proceed with the steps below 3. Change to `build` directory and run `cmake ..` command to configure your build. Windows users can do the same with cmake-gui application. 4. On Windows, build the solution found in the build directory. On Linux, run `make` from the build directory. -On successful build you will find compiled test and example binaries in `bin` +On successfull build you will find compiled test and example binaries in `bin` directory. The generated documentation will be available in `doc/html` directory of the build tree. To run tests after finished build please run `make test` or `ctest` from your build tree. You can get detailed output using `ctest @@ -136,25 +126,4 @@ The following diagram shows the process. ![simpledom](doc/diagram/simpledom.png) -More [examples](https://github.com/Tencent/rapidjson/tree/master/example) are available: - -* DOM API - * [tutorial](https://github.com/Tencent/rapidjson/blob/master/example/tutorial/tutorial.cpp): Basic usage of DOM API. - -* SAX API - * [simplereader](https://github.com/Tencent/rapidjson/blob/master/example/simplereader/simplereader.cpp): Dumps all SAX events while parsing a JSON by `Reader`. - * [condense](https://github.com/Tencent/rapidjson/blob/master/example/condense/condense.cpp): A command line tool to rewrite a JSON, with all whitespaces removed. - * [pretty](https://github.com/Tencent/rapidjson/blob/master/example/pretty/pretty.cpp): A command line tool to rewrite a JSON with indents and newlines by `PrettyWriter`. - * [capitalize](https://github.com/Tencent/rapidjson/blob/master/example/capitalize/capitalize.cpp): A command line tool to capitalize strings in JSON. - * [messagereader](https://github.com/Tencent/rapidjson/blob/master/example/messagereader/messagereader.cpp): Parse a JSON message with SAX API. - * [serialize](https://github.com/Tencent/rapidjson/blob/master/example/serialize/serialize.cpp): Serialize a C++ object into JSON with SAX API. - * [jsonx](https://github.com/Tencent/rapidjson/blob/master/example/jsonx/jsonx.cpp): Implements a `JsonxWriter` which stringify SAX events into [JSONx](https://www-01.ibm.com/support/knowledgecenter/SS9H2Y_7.1.0/com.ibm.dp.doc/json_jsonx.html) (a kind of XML) format. The example is a command line tool which converts input JSON into JSONx format. - -* Schema - * [schemavalidator](https://github.com/Tencent/rapidjson/blob/master/example/schemavalidator/schemavalidator.cpp) : A command line tool to validate a JSON with a JSON schema. - -* Advanced - * [prettyauto](https://github.com/Tencent/rapidjson/blob/master/example/prettyauto/prettyauto.cpp): A modified version of [pretty](https://github.com/Tencent/rapidjson/blob/master/example/pretty/pretty.cpp) to automatically handle JSON with any UTF encodings. - * [parsebyparts](https://github.com/Tencent/rapidjson/blob/master/example/parsebyparts/parsebyparts.cpp): Implements an `AsyncDocumentParser` which can parse JSON in parts, using C++11 thread. - * [filterkey](https://github.com/Tencent/rapidjson/blob/master/example/filterkey/filterkey.cpp): A command line tool to remove all values with user-specified key. - * [filterkeydom](https://github.com/Tencent/rapidjson/blob/master/example/filterkeydom/filterkeydom.cpp): Same tool as above, but it demonstrates how to use a generator to populate a `Document`. +More [examples](https://github.com/miloyip/rapidjson/tree/master/example) are available. diff --git a/test/unit/TestModelFactory.h b/test/unit/TestModelFactory.h index ca070890d..f848f5536 100644 --- a/test/unit/TestModelFactory.h +++ b/test/unit/TestModelFactory.h @@ -89,7 +89,7 @@ public: scene->mRootNode = new aiNode; scene->mRootNode->mNumMeshes = 1; - scene->mRootNode->mMeshes = new unsigned int[1]{ 0 }; + scene->mRootNode->mMeshes = new unsigned int[scene->mRootNode->mNumMeshes]{ 0 }; return scene; } diff --git a/test/unit/utMetadata.cpp b/test/unit/utMetadata.cpp index 4109b068c..b44de1eb2 100644 --- a/test/unit/utMetadata.cpp +++ b/test/unit/utMetadata.cpp @@ -79,7 +79,6 @@ TEST_F( utMetadata, allocTest ) { EXPECT_EQ( 1U, data->mNumProperties ); EXPECT_NE( nullptr, data->mKeys ); EXPECT_NE( nullptr, data->mValues ); - aiMetadata::Dealloc( data ); } TEST_F( utMetadata, get_set_pod_Test ) { diff --git a/test/unit/utRemoveVCProcess.cpp b/test/unit/utRemoveVCProcess.cpp index 6caa72c11..c78e80d3f 100644 --- a/test/unit/utRemoveVCProcess.cpp +++ b/test/unit/utRemoveVCProcess.cpp @@ -70,7 +70,7 @@ TEST_F( utRevmoveVCProcess, issue1266_ProcessMeshTest_NoCrash ) { mesh->mNumVertices = 1; mesh->mColors[ 0 ] = new aiColor4D[ 2 ]; scene->mMeshes[ 0 ] = mesh; - std::unique_ptr process(new RemoveVCProcess); + RemoveVCProcess *process = new RemoveVCProcess; process->Execute( scene ); delete scene; } From a3053eb35896e8c7122028703d88aa7d1e78ad29 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Wed, 4 Oct 2017 20:50:16 +0300 Subject: [PATCH 239/305] Travis: Build with clang too --- .travis.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.travis.yml b/.travis.yml index d59689f78..2eab656cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,10 @@ branches: os: - linux +compiler: + - gcc + - clang + env: global: # COVERITY_SCAN_TOKEN @@ -34,6 +38,10 @@ env: matrix: exclude: - os: linux + compiler: gcc + env: + - os: linux + compiler: clang env: include: @@ -49,6 +57,12 @@ matrix: - os: linux compiler: gcc env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + - os: linux + compiler: clang + env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + - os: linux + compiler: clang + env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From 5cc316b874ad8976983f5ba89e473ab14c149966 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Wed, 4 Oct 2017 21:10:49 +0300 Subject: [PATCH 240/305] Travis: Treat warnings as errors, without typos this time --- .travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.sh b/.travis.sh index f19c3a000..1ab1ee2b1 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,6 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_ERROR=ON -DASSIMP_ASAN=$ASAN + cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_WERROR=ON -DASSIMP_ASAN=$ASAN } if [ $ANDROID ]; then From d28e88feb772d192dc9c5586eeeea682c3e6a536 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Wed, 4 Oct 2017 21:21:36 +0300 Subject: [PATCH 241/305] CMake: Remove OpenMP stuff, it's unused and breaks Travis clang build --- CMakeLists.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c0fe062e..0aae76837 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,19 +151,6 @@ IF(ASSIMP_DOUBLE_PRECISION) ADD_DEFINITIONS(-DASSIMP_DOUBLE_PRECISION) ENDIF(ASSIMP_DOUBLE_PRECISION) -# Check for OpenMP support -find_package(OpenMP) -if (OPENMP_FOUND) - SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - - IF(MSVC) - IF(MSVC_VERSION GREATER 1910) - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:twoPhase-") - ENDIF() - ENDIF() -endif() - CONFIGURE_FILE( ${CMAKE_CURRENT_LIST_DIR}/revision.h.in ${CMAKE_CURRENT_BINARY_DIR}/revision.h From ae8a4c0c62f43db2126fc7228877346d271a5600 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 10:45:25 +0300 Subject: [PATCH 242/305] Fix warning about non-constant array size --- test/unit/TestModelFactory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/TestModelFactory.h b/test/unit/TestModelFactory.h index f848f5536..ca070890d 100644 --- a/test/unit/TestModelFactory.h +++ b/test/unit/TestModelFactory.h @@ -89,7 +89,7 @@ public: scene->mRootNode = new aiNode; scene->mRootNode->mNumMeshes = 1; - scene->mRootNode->mMeshes = new unsigned int[scene->mRootNode->mNumMeshes]{ 0 }; + scene->mRootNode->mMeshes = new unsigned int[1]{ 0 }; return scene; } From 9f5b58e706221857cd7010710f00e8d6536eb540 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:10:22 +0300 Subject: [PATCH 243/305] Upgrade RapidJSON to get rid of a clang warning --- .../rapidjson/include/rapidjson/allocators.h | 42 +- .../rapidjson/include/rapidjson/document.h | 1073 +++++++-- .../include/rapidjson/encodedstream.h | 58 +- .../rapidjson/include/rapidjson/encodings.h | 233 +- .../rapidjson/include/rapidjson/error/en.h | 21 +- .../rapidjson/include/rapidjson/error/error.h | 27 +- .../include/rapidjson/filereadstream.h | 13 +- .../include/rapidjson/filewritestream.h | 17 +- contrib/rapidjson/include/rapidjson/fwd.h | 151 ++ .../include/rapidjson/internal/biginteger.h | 14 +- .../include/rapidjson/internal/diyfp.h | 19 +- .../include/rapidjson/internal/dtoa.h | 50 +- .../include/rapidjson/internal/ieee754.h | 3 +- .../include/rapidjson/internal/regex.h | 734 ++++++ .../include/rapidjson/internal/stack.h | 68 +- .../include/rapidjson/internal/strfunc.h | 32 +- .../include/rapidjson/internal/strtod.h | 39 +- .../include/rapidjson/internal/swap.h | 9 + .../include/rapidjson/istreamwrapper.h | 115 + .../include/rapidjson/memorybuffer.h | 2 +- .../include/rapidjson/memorystream.h | 16 +- .../include/rapidjson/msinttypes/stdint.h | 8 +- .../include/rapidjson/ostreamwrapper.h | 81 + contrib/rapidjson/include/rapidjson/pointer.h | 131 +- .../include/rapidjson/prettywriter.h | 98 +- .../rapidjson/include/rapidjson/rapidjson.h | 314 ++- contrib/rapidjson/include/rapidjson/reader.h | 1345 ++++++++--- contrib/rapidjson/include/rapidjson/schema.h | 2016 +++++++++++++++++ contrib/rapidjson/include/rapidjson/stream.h | 179 ++ .../include/rapidjson/stringbuffer.h | 30 +- contrib/rapidjson/include/rapidjson/writer.h | 468 +++- contrib/rapidjson/license.txt | 2 +- contrib/rapidjson/readme.md | 81 +- 33 files changed, 6467 insertions(+), 1022 deletions(-) create mode 100644 contrib/rapidjson/include/rapidjson/fwd.h create mode 100644 contrib/rapidjson/include/rapidjson/internal/regex.h create mode 100644 contrib/rapidjson/include/rapidjson/istreamwrapper.h create mode 100644 contrib/rapidjson/include/rapidjson/ostreamwrapper.h create mode 100644 contrib/rapidjson/include/rapidjson/schema.h create mode 100644 contrib/rapidjson/include/rapidjson/stream.h diff --git a/contrib/rapidjson/include/rapidjson/allocators.h b/contrib/rapidjson/include/rapidjson/allocators.h index d74a67155..655f4a385 100644 --- a/contrib/rapidjson/include/rapidjson/allocators.h +++ b/contrib/rapidjson/include/rapidjson/allocators.h @@ -179,9 +179,10 @@ public: size = RAPIDJSON_ALIGN(size); if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) - AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); + if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) + return NULL; - void *buffer = reinterpret_cast(chunkHead_ + 1) + chunkHead_->size; + void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; chunkHead_->size += size; return buffer; } @@ -194,14 +195,16 @@ public: if (newSize == 0) return NULL; + originalSize = RAPIDJSON_ALIGN(originalSize); + newSize = RAPIDJSON_ALIGN(newSize); + // Do not shrink if new size is smaller than original if (originalSize >= newSize) return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { + if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { size_t increment = static_cast(newSize - originalSize); - increment = RAPIDJSON_ALIGN(increment); if (chunkHead_->size + increment <= chunkHead_->capacity) { chunkHead_->size += increment; return originalPtr; @@ -209,11 +212,13 @@ public: } // Realloc process: allocate and copy memory, do not free original buffer. - void* newBuffer = Malloc(newSize); - RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. - if (originalSize) - std::memcpy(newBuffer, originalPtr, originalSize); - return newBuffer; + if (void* newBuffer = Malloc(newSize)) { + if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; + } + else + return NULL; } //! Frees a memory block (concept Allocator) @@ -227,15 +232,20 @@ private: //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. + \return true if success. */ - void AddChunk(size_t capacity) { + bool AddChunk(size_t capacity) { if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); - ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity)); - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); + if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + return true; + } + else + return false; } static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. diff --git a/contrib/rapidjson/include/rapidjson/document.h b/contrib/rapidjson/include/rapidjson/document.h index c6acbd907..93b091f64 100644 --- a/contrib/rapidjson/include/rapidjson/document.h +++ b/contrib/rapidjson/include/rapidjson/document.h @@ -20,40 +20,29 @@ #include "reader.h" #include "internal/meta.h" #include "internal/strfunc.h" +#include "memorystream.h" +#include "encodedstream.h" #include // placement new +#include +RAPIDJSON_DIAG_PUSH #ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -#elif defined(__GNUC__) -RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +#ifdef __GNUC__ RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 6 +RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions #endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_HAS_STDSTRING - -#ifndef RAPIDJSON_HAS_STDSTRING -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation -#else -#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default -#endif -/*! \def RAPIDJSON_HAS_STDSTRING - \ingroup RAPIDJSON_CONFIG - \brief Enable RapidJSON support for \c std::string - - By defining this preprocessor symbol to \c 1, several convenience functions for using - \ref rapidjson::GenericValue with \c std::string are enabled, especially - for construction and comparison. - - \hideinitializer -*/ -#endif // !defined(RAPIDJSON_HAS_STDSTRING) - -#if RAPIDJSON_HAS_STDSTRING -#include -#endif // RAPIDJSON_HAS_STDSTRING +#endif // __GNUC__ #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::iterator, std::random_access_iterator_tag @@ -69,6 +58,9 @@ RAPIDJSON_NAMESPACE_BEGIN template class GenericValue; +template +class GenericDocument; + //! Name-value pair in a JSON object value. /*! This class was internal to GenericValue. It used to be a inner struct. @@ -155,6 +147,7 @@ public: Otherwise, the copy constructor is implicitly defined. */ GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } //! @name stepping //@{ @@ -257,6 +250,7 @@ struct GenericStringRef { typedef CharType Ch; //!< character type of the string //! Create string reference from \c const character array +#ifndef __clang__ // -Wdocumentation /*! This constructor implicitly creates a constant string reference from a \c const character array. It has better performance than @@ -279,11 +273,13 @@ struct GenericStringRef { In such cases, the referenced string should be \b copied to the GenericValue instead. */ +#endif template GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT : s(str), length(N-1) {} //! Explicitly create string reference from \c const character pointer +#ifndef __clang__ // -Wdocumentation /*! This constructor can be used to \b explicitly create a reference to a constant string pointer. @@ -302,18 +298,23 @@ struct GenericStringRef { In such cases, the referenced string should be \b copied to the GenericValue instead. */ +#endif explicit GenericStringRef(const CharType* str) - : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != NULL); } + : s(str), length(NotNullStrLen(str)) {} //! Create constant string reference from pointer and length +#ifndef __clang__ // -Wdocumentation /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \param len length of the string, excluding the trailing NULL terminator \post \ref s == str && \ref length == len \note Constant complexity. */ +#endif GenericStringRef(const CharType* str, SizeType len) - : s(str), length(len) { RAPIDJSON_ASSERT(s != NULL); } + : s(RAPIDJSON_LIKELY(str) ? str : emptyString), length(len) { RAPIDJSON_ASSERT(str != 0 || len == 0u); } + + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} //! implicit conversion to plain CharType pointer operator const Ch *() const { return s; } @@ -322,13 +323,24 @@ struct GenericStringRef { const SizeType length; //!< length of the string (excluding the trailing NULL terminator) private: - //! Disallow copy-assignment - GenericStringRef operator=(const GenericStringRef&); + SizeType NotNullStrLen(const CharType* str) { + RAPIDJSON_ASSERT(str != 0); + return internal::StrLen(str); + } + + /// Empty string - used when passing in a NULL pointer + static const Ch emptyString[]; + //! Disallow construction from non-const array template GenericStringRef(CharType (&str)[N]) /* = delete */; + //! Copy assignment operator not permitted - immutable type + GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */; }; +template +const CharType GenericStringRef::emptyString[] = { CharType() }; + //! Mark a character pointer as constant string /*! Mark a plain character pointer as a "string literal". This function can be used to avoid copying a character string to be referenced as a @@ -343,7 +355,7 @@ private: */ template inline GenericStringRef StringRef(const CharType* str) { - return GenericStringRef(str, internal::StrLen(str)); + return GenericStringRef(str); } //! Mark a character pointer as constant string @@ -401,6 +413,127 @@ template struct IsGenericValue : IsGenericValueImpl::Type {}; } // namespace internal +/////////////////////////////////////////////////////////////////////////////// +// TypeHelper + +namespace internal { + +template +struct TypeHelper {}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsBool(); } + static bool Get(const ValueType& v) { return v.GetBool(); } + static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } + static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static int Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt64(); } + static int64_t Get(const ValueType& v) { return v.GetInt64(); } + static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } + static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint64(); } + static uint64_t Get(const ValueType& v) { return v.GetUint64(); } + static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } + static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsDouble(); } + static double Get(const ValueType& v) { return v.GetDouble(); } + static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } + static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsFloat(); } + static float Get(const ValueType& v) { return v.GetFloat(); } + static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } + static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } +}; + +template +struct TypeHelper { + typedef const typename ValueType::Ch* StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } + static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; + +#if RAPIDJSON_HAS_STDSTRING +template +struct TypeHelper > { + typedef std::basic_string StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } + static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; +#endif + +template +struct TypeHelper { + typedef typename ValueType::Array ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(ValueType& v) { return v.GetArray(); } + static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } + static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstArray ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(const ValueType& v) { return v.GetArray(); } +}; + +template +struct TypeHelper { + typedef typename ValueType::Object ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(ValueType& v) { return v.GetObject(); } + static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstObject ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(const ValueType& v) { return v.GetObject(); } +}; + +} // namespace internal + +// Forward declarations +template class GenericArray; +template class GenericObject; + /////////////////////////////////////////////////////////////////////////////// // GenericValue @@ -428,17 +561,21 @@ public: typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. typedef GenericValue ValueType; //!< Value type of itself. + typedef GenericArray Array; + typedef GenericArray ConstArray; + typedef GenericObject Object; + typedef GenericObject ConstObject; //!@name Constructors and destructor. //@{ //! Default constructor creates a null value. - GenericValue() RAPIDJSON_NOEXCEPT : data_(), flags_(kNullFlag) {} + GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 - GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_), flags_(rhs.flags_) { - rhs.flags_ = kNullFlag; // give up contents + GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { + rhs.data_.f.flags = kNullFlag; // give up contents } #endif @@ -446,6 +583,16 @@ private: //! Copy constructor is not permitted. GenericValue(const GenericValue& rhs); +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Moving from a GenericDocument is not permitted. + template + GenericValue(GenericDocument&& rhs); + + //! Move assignment from a GenericDocument is not permitted. + template + GenericValue& operator=(GenericDocument&& rhs); +#endif + public: //! Constructor with JSON value type. @@ -453,13 +600,13 @@ public: \param type Type of the value. \note Default content for number is zero. */ - explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_(), flags_() { - static const unsigned defaultFlags[7] = { + explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { + static const uint16_t defaultFlags[7] = { kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, kNumberAnyFlag }; - RAPIDJSON_ASSERT(type <= kNumberType); - flags_ = defaultFlags[type]; + RAPIDJSON_ASSERT(type >= kNullType && type <= kNumberType); + data_.f.flags = defaultFlags[type]; // Use ShortString to store empty string. if (type == kStringType) @@ -471,10 +618,50 @@ public: \tparam SourceAllocator allocator of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) \see CopyFrom() */ - template< typename SourceAllocator > - GenericValue(const GenericValue& rhs, Allocator & allocator); + template + GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + switch (rhs.GetType()) { + case kObjectType: { + SizeType count = rhs.data_.o.size; + Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); + } + data_.f.flags = kObjectFlag; + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } + break; + case kArrayType: { + SizeType count = rhs.data_.a.size; + GenericValue* le = reinterpret_cast(allocator.Malloc(count * sizeof(GenericValue))); + const GenericValue* re = rhs.GetElementsPointer(); + for (SizeType i = 0; i < count; i++) + new (&le[i]) GenericValue(re[i], allocator, copyConstStrings); + data_.f.flags = kArrayFlag; + data_.a.size = data_.a.capacity = count; + SetElementsPointer(le); + } + break; + case kStringType: + if (rhs.data_.f.flags == kConstStringFlag && !copyConstStrings) { + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + } + else + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } + } //! Constructor for boolean value. /*! \param b Boolean value @@ -484,96 +671,125 @@ public: */ #ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen template - explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 #else explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT #endif - : data_(), flags_(b ? kTrueFlag : kFalseFlag) { + : data_() { // safe-guard against failing SFINAE RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + data_.f.flags = b ? kTrueFlag : kFalseFlag; } //! Constructor for int value. - explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberIntFlag) { + explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { data_.n.i64 = i; - if (i >= 0) - flags_ |= kUintFlag | kUint64Flag; + data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; } //! Constructor for unsigned value. - explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUintFlag) { + explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { data_.n.u64 = u; - if (!(u & 0x80000000)) - flags_ |= kIntFlag | kInt64Flag; + data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); } //! Constructor for int64_t value. - explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberInt64Flag) { + explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { data_.n.i64 = i64; + data_.f.flags = kNumberInt64Flag; if (i64 >= 0) { - flags_ |= kNumberUint64Flag; + data_.f.flags |= kNumberUint64Flag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - flags_ |= kUintFlag; + data_.f.flags |= kUintFlag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; + data_.f.flags |= kIntFlag; } else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; + data_.f.flags |= kIntFlag; } //! Constructor for uint64_t value. - explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberUint64Flag) { + explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { data_.n.u64 = u64; + data_.f.flags = kNumberUint64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) - flags_ |= kInt64Flag; + data_.f.flags |= kInt64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - flags_ |= kUintFlag; + data_.f.flags |= kUintFlag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - flags_ |= kIntFlag; + data_.f.flags |= kIntFlag; } //! Constructor for double value. - explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_(), flags_(kNumberDoubleFlag) { data_.n.d = d; } + explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for float value. + explicit GenericValue(float f) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = static_cast(f); data_.f.flags = kNumberDoubleFlag; } //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(StringRef(s, length)); } + GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } //! Constructor for constant string (i.e. do not make a copy of string) - explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_(), flags_() { SetStringRaw(s); } + explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s, length), allocator); } + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } + GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Constructor for copy-string from a string object (i.e. do make a copy of string) /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue(const std::basic_string& s, Allocator& allocator) : data_(), flags_() { SetStringRaw(StringRef(s), allocator); } + GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #endif + //! Constructor for Array. + /*! + \param a An array obtained by \c GetArray(). + \note \c Array is always pass-by-value. + \note the source array is moved into this value and the sourec array becomes empty. + */ + GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { + a.value_.data_ = Data(); + a.value_.data_.f.flags = kArrayFlag; + } + + //! Constructor for Object. + /*! + \param o An object obtained by \c GetObject(). + \note \c Object is always pass-by-value. + \note the source object is moved into this value and the sourec object becomes empty. + */ + GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { + o.value_.data_ = Data(); + o.value_.data_.f.flags = kObjectFlag; + } + //! Destructor. /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(flags_) { + switch(data_.f.flags) { case kArrayFlag: - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(data_.a.elements); + { + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(e); + } break; case kObjectFlag: for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) m->~Member(); - Allocator::Free(data_.o.members); + Allocator::Free(GetMembersPointer()); break; case kCopyStringFlag: - Allocator::Free(const_cast(data_.s.str)); + Allocator::Free(const_cast(GetStringPointer())); break; default: @@ -638,12 +854,13 @@ public: \tparam SourceAllocator Allocator type of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator to use for copying + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) */ template - GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { - RAPIDJSON_ASSERT((void*)this != (void const*)&rhs); + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); this->~GenericValue(); - new (this) GenericValue(rhs, allocator); + new (this) GenericValue(rhs, allocator, copyConstStrings); return *this; } @@ -660,6 +877,20 @@ public: return *this; } + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.value, b.value); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + //! Prepare Value for move semantics /*! \return *this */ GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } @@ -709,7 +940,7 @@ public: else return data_.n.u64 == rhs.data_.n.u64; - default: // kTrueType, kFalseType, kNullType + default: return true; } } @@ -757,20 +988,58 @@ public: //!@name Type //@{ - Type GetType() const { return static_cast(flags_ & kTypeMask); } - bool IsNull() const { return flags_ == kNullFlag; } - bool IsFalse() const { return flags_ == kFalseFlag; } - bool IsTrue() const { return flags_ == kTrueFlag; } - bool IsBool() const { return (flags_ & kBoolFlag) != 0; } - bool IsObject() const { return flags_ == kObjectFlag; } - bool IsArray() const { return flags_ == kArrayFlag; } - bool IsNumber() const { return (flags_ & kNumberFlag) != 0; } - bool IsInt() const { return (flags_ & kIntFlag) != 0; } - bool IsUint() const { return (flags_ & kUintFlag) != 0; } - bool IsInt64() const { return (flags_ & kInt64Flag) != 0; } - bool IsUint64() const { return (flags_ & kUint64Flag) != 0; } - bool IsDouble() const { return (flags_ & kDoubleFlag) != 0; } - bool IsString() const { return (flags_ & kStringFlag) != 0; } + Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } + bool IsNull() const { return data_.f.flags == kNullFlag; } + bool IsFalse() const { return data_.f.flags == kFalseFlag; } + bool IsTrue() const { return data_.f.flags == kTrueFlag; } + bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } + bool IsObject() const { return data_.f.flags == kObjectFlag; } + bool IsArray() const { return data_.f.flags == kArrayFlag; } + bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } + bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } + bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } + bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } + bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } + bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } + bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } + + // Checks whether a number can be losslessly converted to a double. + bool IsLosslessDouble() const { + if (!IsNumber()) return false; + if (IsUint64()) { + uint64_t u = GetUint64(); + volatile double d = static_cast(u); + return (d >= 0.0) + && (d < static_cast((std::numeric_limits::max)())) + && (u == static_cast(d)); + } + if (IsInt64()) { + int64_t i = GetInt64(); + volatile double d = static_cast(i); + return (d >= static_cast((std::numeric_limits::min)())) + && (d < static_cast((std::numeric_limits::max)())) + && (i == static_cast(d)); + } + return true; // double, int, uint are always lossless + } + + // Checks whether a number is a float (possible lossy). + bool IsFloat() const { + if ((data_.f.flags & kDoubleFlag) == 0) + return false; + double d = GetDouble(); + return d >= -3.4028234e38 && d <= 3.4028234e38; + } + // Checks whether a number can be losslessly converted to a float. + bool IsLosslessFloat() const { + if (!IsNumber()) return false; + double a = GetDouble(); + if (a < static_cast(-(std::numeric_limits::max)()) + || a > static_cast((std::numeric_limits::max)())) + return false; + double b = static_cast(static_cast(a)); + return a >= b && a <= b; // Prevent -Wfloat-equal + } //@} @@ -784,7 +1053,7 @@ public: //!@name Bool //@{ - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return flags_ == kTrueFlag; } + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } //!< Set boolean value /*! \post IsBool() == true */ GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } @@ -837,8 +1106,14 @@ public: return member->value; else { RAPIDJSON_ASSERT(false); // see above note - static GenericValue NullValue; - return NullValue; + + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); } } template @@ -852,16 +1127,16 @@ public: //! Const member iterator /*! \pre IsObject() == true */ - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members); } + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } //! Const \em past-the-end member iterator /*! \pre IsObject() == true */ - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(data_.o.members + data_.o.size); } + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } //! Member iterator /*! \pre IsObject() == true */ - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members); } + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } //! \em Past-the-end member iterator /*! \pre IsObject() == true */ - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(data_.o.members + data_.o.size); } + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } //! Check whether a member exists in the object. /*! @@ -949,8 +1224,8 @@ public: \return Iterator to member, if it exists. Otherwise returns \ref MemberEnd(). */ - MemberIterator FindMember(const std::basic_string& name) { return FindMember(StringRef(name)); } - ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(StringRef(name)); } + MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } #endif //! Add a member (name-value pair) to the object. @@ -967,20 +1242,21 @@ public: RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - Object& o = data_.o; + ObjectData& o = data_.o; if (o.size >= o.capacity) { if (o.capacity == 0) { o.capacity = kDefaultObjectCapacity; - o.members = reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member))); + SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); } else { SizeType oldCapacity = o.capacity; o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - o.members = reinterpret_cast(allocator.Realloc(o.members, oldCapacity * sizeof(Member), o.capacity * sizeof(Member))); + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); } } - o.members[o.size].name.RawAssign(name); - o.members[o.size].value.RawAssign(value); + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); o.size++; return *this; } @@ -1159,18 +1435,14 @@ public: MemberIterator RemoveMember(MemberIterator m) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - MemberIterator last(data_.o.members + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) { - // Move the last one to this place - *m = *last; - } - else { - // Only one left, just destroy - m->~Member(); - } + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy --data_.o.size; return m; } @@ -1200,7 +1472,7 @@ public: MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(data_.o.members != 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); @@ -1208,11 +1480,39 @@ public: MemberIterator pos = MemberBegin() + (first - MemberBegin()); for (MemberIterator itr = pos; itr != last; ++itr) itr->~Member(); - std::memmove(&*pos, &*last, (MemberEnd() - last) * sizeof(Member)); - data_.o.size -= (last - first); + std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); return pos; } + //! Erase a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note Linear time complexity. + */ + bool EraseMember(const Ch* name) { + GenericValue n(StringRef(name)); + return EraseMember(n); + } + +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } +#endif + + template + bool EraseMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + EraseMember(m); + return true; + } + else + return false; + } + + Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + //@} //!@name Array @@ -1220,7 +1520,7 @@ public: //! Set this value as an empty array. /*! \post IsArray == true */ - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } //! Get the number of elements in array. SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } @@ -1237,8 +1537,9 @@ public: */ void Clear() { RAPIDJSON_ASSERT(IsArray()); - for (SizeType i = 0; i < data_.a.size; ++i) - data_.a.elements[i].~GenericValue(); + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); data_.a.size = 0; } @@ -1250,16 +1551,16 @@ public: GenericValue& operator[](SizeType index) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(index < data_.a.size); - return data_.a.elements[index]; + return GetElementsPointer()[index]; } const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } //! Element iterator /*! \pre IsArray() == true */ - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements; } + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } //! \em Past-the-end element iterator /*! \pre IsArray() == true */ - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return data_.a.elements + data_.a.size; } + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } //! Constant element iterator /*! \pre IsArray() == true */ ConstValueIterator Begin() const { return const_cast(*this).Begin(); } @@ -1276,7 +1577,7 @@ public: GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsArray()); if (newCapacity > data_.a.capacity) { - data_.a.elements = (GenericValue*)allocator.Realloc(data_.a.elements, data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)); + SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); data_.a.capacity = newCapacity; } return *this; @@ -1296,7 +1597,7 @@ public: RAPIDJSON_ASSERT(IsArray()); if (data_.a.size >= data_.a.capacity) Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); - data_.a.elements[data_.a.size++].RawAssign(value); + GetElementsPointer()[data_.a.size++].RawAssign(value); return *this; } @@ -1350,7 +1651,7 @@ public: GenericValue& PopBack() { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(!Empty()); - data_.a.elements[--data_.a.size].~GenericValue(); + GetElementsPointer()[--data_.a.size].~GenericValue(); return *this; } @@ -1376,35 +1677,48 @@ public: ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(data_.a.size > 0); - RAPIDJSON_ASSERT(data_.a.elements != 0); + RAPIDJSON_ASSERT(GetElementsPointer() != 0); RAPIDJSON_ASSERT(first >= Begin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= End()); ValueIterator pos = Begin() + (first - Begin()); for (ValueIterator itr = pos; itr != last; ++itr) itr->~GenericValue(); - std::memmove(pos, last, (End() - last) * sizeof(GenericValue)); - data_.a.size -= (last - first); + std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); return pos; } + Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } + ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } + //@} //!@name Number //@{ - int GetInt() const { RAPIDJSON_ASSERT(flags_ & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(flags_ & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(flags_ & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(flags_ & kUint64Flag); return data_.n.u64; } + int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + //! Get the value as double type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. + */ double GetDouble() const { RAPIDJSON_ASSERT(IsNumber()); - if ((flags_ & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((flags_ & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((flags_ & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((flags_ & kInt64Flag) != 0) return (double)data_.n.i64; // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((flags_ & kUint64Flag) != 0); return (double)data_.n.u64; // uint64_t -> double (may lose precision) + if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) + } + + //! Get the value as float type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. + */ + float GetFloat() const { + return static_cast(GetDouble()); } GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } @@ -1412,18 +1726,19 @@ public: GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(static_cast(f)); return *this; } //@} //!@name String //@{ - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? data_.ss.str : data_.s.str); } + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((flags_ & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. @@ -1450,7 +1765,7 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { return SetString(StringRef(s, length), allocator); } //! Set this value as a string by copying from source string. /*! \param s source string. @@ -1458,7 +1773,15 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(StringRef(s), allocator); } + + //! Set this value as a string by copying from source string. + /*! \param s source string reference + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(StringRefType s, Allocator& allocator) { this->~GenericValue(); SetStringRaw(s, allocator); return *this; } #if RAPIDJSON_HAS_STDSTRING //! Set this value as a string by copying from source string. @@ -1468,11 +1791,35 @@ public: \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(StringRef(s), allocator); } #endif //@} + //!@name Array + //@{ + + //! Templated version for checking whether this value is type T. + /*! + \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string + */ + template + bool Is() const { return internal::TypeHelper::Is(*this); } + + template + T Get() const { return internal::TypeHelper::Get(*this); } + + template + T Get() { return internal::TypeHelper::Get(*this); } + + template + ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } + + template + ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } + + //@} + //! Generate events of this value to a Handler. /*! This function adopts the GoF visitor pattern. Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. @@ -1488,35 +1835,35 @@ public: case kTrueType: return handler.Bool(true); case kObjectType: - if (!handler.StartObject()) + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) return false; for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. - if (!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.flags_ & kCopyFlag) != 0)) + if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) return false; - if (!m->value.Accept(handler)) + if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) return false; } return handler.EndObject(data_.o.size); case kArrayType: - if (!handler.StartArray()) + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) return false; - for (GenericValue* v = data_.a.elements; v != data_.a.elements + data_.a.size; ++v) - if (!v->Accept(handler)) + for (const GenericValue* v = Begin(); v != End(); ++v) + if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) return false; return handler.EndArray(data_.a.size); case kStringType: - return handler.String(GetString(), GetStringLength(), (flags_ & kCopyFlag) != 0); + return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); default: RAPIDJSON_ASSERT(GetType() == kNumberType); - if (IsInt()) return handler.Int(data_.n.i.i); + if (IsDouble()) return handler.Double(data_.n.d); + else if (IsInt()) return handler.Int(data_.n.i.i); else if (IsUint()) return handler.Uint(data_.n.u.u); else if (IsInt64()) return handler.Int64(data_.n.i64); - else if (IsUint64()) return handler.Uint64(data_.n.u64); - else return handler.Double(data_.n.d); + else return handler.Uint64(data_.n.u64); } } @@ -1525,16 +1872,16 @@ private: template friend class GenericDocument; enum { - kBoolFlag = 0x100, - kNumberFlag = 0x200, - kIntFlag = 0x400, - kUintFlag = 0x800, - kInt64Flag = 0x1000, - kUint64Flag = 0x2000, - kDoubleFlag = 0x4000, - kStringFlag = 0x100000, - kCopyFlag = 0x200000, - kInlineStrFlag = 0x400000, + kBoolFlag = 0x0008, + kNumberFlag = 0x0010, + kIntFlag = 0x0020, + kUintFlag = 0x0040, + kInt64Flag = 0x0080, + kUint64Flag = 0x0100, + kDoubleFlag = 0x0200, + kStringFlag = 0x0400, + kCopyFlag = 0x0800, + kInlineStrFlag = 0x1000, // Initial flags of different types. kNullFlag = kNullType, @@ -1552,16 +1899,27 @@ private: kObjectFlag = kObjectType, kArrayFlag = kArrayType, - kTypeMask = 0xFF // bitwise-and with mask of 0xFF can be optimized by compiler + kTypeMask = 0x07 }; static const SizeType kDefaultArrayCapacity = 16; static const SizeType kDefaultObjectCapacity = 16; + struct Flag { +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION + char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer +#elif RAPIDJSON_64BIT + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes +#else + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes +#endif + uint16_t flags; + }; + struct String { - const Ch* str; SizeType length; - unsigned hashcode; //!< reserved + SizeType hashcode; //!< reserved + const Ch* str; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars @@ -1570,15 +1928,15 @@ private: // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as // the string terminator as well. For getting the string length back from that value just use // "MaxSize - str[LenPos]". - // This allows to store 11-chars strings in 32-bit mode and 15-chars strings in 64-bit mode - // inline (for `UTF8`-encoded strings). + // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, + // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). struct ShortString { - enum { MaxChars = sizeof(String) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; Ch str[MaxChars]; - inline static bool Usable(SizeType len) { return (MaxSize >= len); } - inline void SetLength(SizeType len) { str[LenPos] = (Ch)(MaxSize - len); } - inline SizeType GetLength() const { return (SizeType)(MaxSize - str[LenPos]); } + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } + inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // By using proper binary layout, retrieval of different integer types do not need conversions. @@ -1607,69 +1965,79 @@ private: double d; }; // 8 bytes - struct Object { - Member* members; + struct ObjectData { SizeType size; SizeType capacity; + Member* members; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - struct Array { - GenericValue* elements; + struct ArrayData { SizeType size; SizeType capacity; + GenericValue* elements; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode union Data { String s; ShortString ss; Number n; - Object o; - Array a; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + ObjectData o; + ArrayData a; + Flag f; + }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } + RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } + RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } + RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } + RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } + RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { - flags_ = kArrayFlag; + data_.f.flags = kArrayFlag; if (count) { - data_.a.elements = (GenericValue*)allocator.Malloc(count * sizeof(GenericValue)); - std::memcpy(data_.a.elements, values, count * sizeof(GenericValue)); + GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); + std::memcpy(e, values, count * sizeof(GenericValue)); } else - data_.a.elements = NULL; + SetElementsPointer(0); data_.a.size = data_.a.capacity = count; } //! Initialize this value as object with initial data, without calling destructor. void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { - flags_ = kObjectFlag; + data_.f.flags = kObjectFlag; if (count) { - data_.o.members = (Member*)allocator.Malloc(count * sizeof(Member)); - std::memcpy(data_.o.members, members, count * sizeof(Member)); + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); + std::memcpy(m, members, count * sizeof(Member)); } else - data_.o.members = NULL; + SetMembersPointer(0); data_.o.size = data_.o.capacity = count; } //! Initialize this value as constant string, without calling destructor. void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { - flags_ = kConstStringFlag; - data_.s.str = s; + data_.f.flags = kConstStringFlag; + SetStringPointer(s); data_.s.length = s.length; } //! Initialize this value as copy string with initial data, without calling destructor. void SetStringRaw(StringRefType s, Allocator& allocator) { - Ch* str = NULL; - if(ShortString::Usable(s.length)) { - flags_ = kShortStringFlag; + Ch* str = 0; + if (ShortString::Usable(s.length)) { + data_.f.flags = kShortStringFlag; data_.ss.SetLength(s.length); str = data_.ss.str; } else { - flags_ = kCopyStringFlag; + data_.f.flags = kCopyStringFlag; data_.s.length = s.length; - str = (Ch *)allocator.Malloc((s.length + 1) * sizeof(Ch)); - data_.s.str = str; + str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); + SetStringPointer(str); } std::memcpy(str, s, s.length * sizeof(Ch)); str[s.length] = '\0'; @@ -1678,8 +2046,8 @@ private: //! Assignment without calling destructor void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { data_ = rhs.data_; - flags_ = rhs.flags_; - rhs.flags_ = kNullFlag; + // data_.f.flags = rhs.data_.f.flags; + rhs.data_.f.flags = kNullFlag; } template @@ -1699,7 +2067,6 @@ private: } Data data_; - unsigned flags_; }; //! GenericValue with UTF8 encoding @@ -1724,7 +2091,22 @@ public: typedef Allocator AllocatorType; //!< Allocator type from template parameter. //! Constructor - /*! \param allocator Optional allocator for allocating memory. + /*! Creates an empty document of specified type. + \param type Mandatory type of object to create. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + } + + //! Constructor + /*! Creates an empty document which type is Null. + \param allocator Optional allocator for allocating memory. \param stackCapacity Optional initial capacity of stack in bytes. \param stackAllocator Optional allocator for allocating memory for stack. */ @@ -1732,13 +2114,13 @@ public: allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - : ValueType(std::move(rhs)), + : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), stack_(std::move(rhs.stack_)), @@ -1778,6 +2160,54 @@ public: } #endif + //! Exchange the contents of this document with those of another. + /*! + \param rhs Another document. + \note Constant complexity. + \see GenericValue::Swap + */ + GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { + ValueType::Swap(rhs); + stack_.Swap(rhs.stack_); + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(parseResult_, rhs.parseResult_); + return *this; + } + + // Allow Swap with ValueType. + // Refer to Effective C++ 3rd Edition/Item 33: Avoid hiding inherited names. + using ValueType::Swap; + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.doc, b.doc); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Populate this document by a generator which produces SAX events. + /*! \tparam Generator A functor with bool f(Handler) prototype. + \param g Generator functor which sends SAX events to the parameter. + \return The document itself for fluent API. + */ + template + GenericDocument& Populate(Generator& g) { + ClearStackOnExit scope(*this); + if (g(*this)) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + //!@name Parse from stream //!@{ @@ -1790,13 +2220,13 @@ public: */ template GenericDocument& ParseStream(InputStream& is) { - ValueType::SetNull(); // Remove existing root if exist - GenericReader reader(&stack_.GetAllocator()); + GenericReader reader( + stack_.HasAllocator() ? &stack_.GetAllocator() : 0); ClearStackOnExit scope(*this); parseResult_ = reader.template Parse(is, *this); if (parseResult_) { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - this->RawAssign(*stack_.template Pop(1)); // Add this-> to prevent issue 13. + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document } return *this; } @@ -1855,7 +2285,7 @@ public: \param str Read-only zero-terminated string to be parsed. */ template - GenericDocument& Parse(const Ch* str) { + GenericDocument& Parse(const typename SourceEncoding::Ch* str) { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); GenericStringStream s(str); return ParseStream(s); @@ -1876,6 +2306,42 @@ public: GenericDocument& Parse(const Ch* str) { return Parse(str); } + + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + MemoryStream ms(reinterpret_cast(str), length * sizeof(typename SourceEncoding::Ch)); + EncodedInputStream is(ms); + ParseStream(is); + return *this; + } + + template + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + template + GenericDocument& Parse(const std::basic_string& str) { + // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) + return Parse(str.c_str()); + } + + template + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str.c_str()); + } + + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str); + } +#endif // RAPIDJSON_HAS_STDSTRING + //!@} //!@name Handling parse errors @@ -1890,10 +2356,26 @@ public: //! Get the position of last parsing error in input, 0 otherwise. size_t GetErrorOffset() const { return parseResult_.Offset(); } + //! Implicit conversion to get the last parse result +#ifndef __clang // -Wdocumentation + /*! \return \ref ParseResult of the last parse operation + + \code + Document doc; + ParseResult ok = doc.Parse(json); + if (!ok) + printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); + \endcode + */ +#endif + operator ParseResult() const { return parseResult_; } //!@} //! Get the allocator of this document. - Allocator& GetAllocator() { return *allocator_; } + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } //! Get the capacity of stack in bytes. size_t GetStackCapacity() const { return stack_.GetCapacity(); } @@ -1910,9 +2392,10 @@ private: }; // callers of the following private Handler functions - template friend class GenericReader; // for parsing + // template friend class GenericReader; // for parsing template friend class GenericValue; // for deep copying +public: // Implementation of Handler bool Null() { new (stack_.template Push()) ValueType(); return true; } bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } @@ -1922,6 +2405,14 @@ private: bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + bool RawNumber(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + bool String(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); @@ -1936,7 +2427,7 @@ private: bool EndObject(SizeType memberCount) { typename ValueType::Member* members = stack_.template Pop(memberCount); - stack_.template Top()->SetObjectRaw(members, (SizeType)memberCount, GetAllocator()); + stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); return true; } @@ -1977,38 +2468,146 @@ private: //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; -// defined here due to the dependency on GenericDocument -template -template -inline -GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) -{ - switch (rhs.GetType()) { - case kObjectType: - case kArrayType: { // perform deep copy via SAX Handler - GenericDocument d(&allocator); - rhs.Accept(d); - RawAssign(*d.stack_.template Pop(1)); - } - break; - case kStringType: - if (rhs.flags_ == kConstStringFlag) { - flags_ = rhs.flags_; - data_ = *reinterpret_cast(&rhs.data_); - } else { - SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); - } - break; - default: // kNumberType, kTrueType, kFalseType, kNullType - flags_ = rhs.flags_; - data_ = *reinterpret_cast(&rhs.data_); - } -} +//! Helper class for accessing Value of array type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetArray(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericArray { +public: + typedef GenericArray ConstArray; + typedef GenericArray Array; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef ValueType* ValueIterator; // This may be const or non-const iterator + typedef const ValueT* ConstValueIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; -RAPIDJSON_NAMESPACE_END + template + friend class GenericValue; -#if defined(_MSC_VER) || defined(__GNUC__) -RAPIDJSON_DIAG_POP + GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} + GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } + ~GenericArray() {} + + SizeType Size() const { return value_.Size(); } + SizeType Capacity() const { return value_.Capacity(); } + bool Empty() const { return value_.Empty(); } + void Clear() const { value_.Clear(); } + ValueType& operator[](SizeType index) const { return value_[index]; } + ValueIterator Begin() const { return value_.Begin(); } + ValueIterator End() const { return value_.End(); } + GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } + GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PopBack() const { value_.PopBack(); return *this; } + ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + ValueIterator begin() const { return value_.Begin(); } + ValueIterator end() const { return value_.End(); } #endif +private: + GenericArray(); + GenericArray(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +//! Helper class for accessing Value of object type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetObject(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericObject { +public: + typedef GenericObject ConstObject; + typedef GenericObject Object; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator + typedef GenericMemberIterator ConstMemberIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename ValueType::Ch Ch; + + template + friend class GenericValue; + + GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} + GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } + ~GenericObject() {} + + SizeType MemberCount() const { return value_.MemberCount(); } + bool ObjectEmpty() const { return value_.ObjectEmpty(); } + template ValueType& operator[](T* name) const { return value_[name]; } + template ValueType& operator[](const GenericValue& name) const { return value_[name]; } +#if RAPIDJSON_HAS_STDSTRING + ValueType& operator[](const std::basic_string& name) const { return value_[name]; } +#endif + MemberIterator MemberBegin() const { return value_.MemberBegin(); } + MemberIterator MemberEnd() const { return value_.MemberEnd(); } + bool HasMember(const Ch* name) const { return value_.HasMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } +#endif + template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } + MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } + template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } +#if RAPIDJSON_HAS_STDSTRING + MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } +#endif + GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_STDSTRING + GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + void RemoveAllMembers() { value_.RemoveAllMembers(); } + bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } +#endif + template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } + MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } + MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } + bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } +#endif + template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + MemberIterator begin() const { return value_.MemberBegin(); } + MemberIterator end() const { return value_.MemberEnd(); } +#endif + +private: + GenericObject(); + GenericObject(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + #endif // RAPIDJSON_DOCUMENT_H_ diff --git a/contrib/rapidjson/include/rapidjson/encodedstream.h b/contrib/rapidjson/include/rapidjson/encodedstream.h index 7c8863fee..223601c05 100644 --- a/contrib/rapidjson/include/rapidjson/encodedstream.h +++ b/contrib/rapidjson/include/rapidjson/encodedstream.h @@ -15,13 +15,19 @@ #ifndef RAPIDJSON_ENCODEDSTREAM_H_ #define RAPIDJSON_ENCODEDSTREAM_H_ -#include "rapidjson.h" +#include "stream.h" +#include "memorystream.h" #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Input byte stream wrapper with a statically bound encoding. @@ -57,10 +63,38 @@ private: Ch current_; }; +//! Specialized for UTF8 MemoryStream. +template <> +class EncodedInputStream, MemoryStream> { +public: + typedef UTF8<>::Ch Ch; + + EncodedInputStream(MemoryStream& is) : is_(is) { + if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); + } + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) {} + void Flush() {} + Ch* PutBegin() { return 0; } + size_t PutEnd(Ch*) { return 0; } + + MemoryStream& is_; + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); +}; + //! Output byte stream wrapper with statically bound encoding. /*! \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam InputByteStream Type of input byte stream. For example, FileWriteStream. + \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. */ template class EncodedOutputStream { @@ -77,8 +111,8 @@ public: void Flush() { os_.Flush(); } // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); } - Ch Take() { RAPIDJSON_ASSERT(false); } + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } @@ -142,11 +176,11 @@ private: // FF FE UTF-16LE // EF BB BF UTF-8 - const unsigned char* c = (const unsigned char *)is_->Peek4(); + const unsigned char* c = reinterpret_cast(is_->Peek4()); if (!c) return; - unsigned bom = c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24); + unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); hasBOM_ = false; if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } @@ -166,7 +200,7 @@ private: // xx xx xx xx UTF-8 if (!hasBOM_) { - unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); switch (pattern) { case 0x08: type_ = kUTF32BE; break; case 0x0A: type_ = kUTF16BE; break; @@ -193,7 +227,7 @@ private: //! Output stream wrapper with dynamically bound encoding and automatic encoding detection. /*! \tparam CharType Type of character for writing. - \tparam InputByteStream type of output byte stream to be wrapped. + \tparam OutputByteStream type of output byte stream to be wrapped. */ template class AutoUTFOutputStream { @@ -227,8 +261,8 @@ public: void Flush() { os_->Flush(); } // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); } - Ch Take() { RAPIDJSON_ASSERT(false); } + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } @@ -254,6 +288,10 @@ private: RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/encodings.h b/contrib/rapidjson/include/rapidjson/encodings.h index 90b46ed32..0df1c3435 100644 --- a/contrib/rapidjson/include/rapidjson/encodings.h +++ b/contrib/rapidjson/include/rapidjson/encodings.h @@ -120,19 +120,45 @@ struct UTF8 { } } + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + PutUnsafe(os, static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + } + template static bool Decode(InputStream& is, unsigned* codepoint) { -#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | ((unsigned char)c & 0x3Fu) -#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) +#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) - Ch c = is.Take(); + typename InputStream::Ch c = is.Take(); if (!(c & 0x80)) { - *codepoint = (unsigned char)c; + *codepoint = static_cast(c); return true; } - unsigned char type = GetRange((unsigned char)c); - *codepoint = (0xFF >> type) & (unsigned char)c; + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) { + *codepoint = 0; + } else { + *codepoint = (0xFFu >> type) & static_cast(c); + } bool result = true; switch (type) { case 2: TAIL(); return result; @@ -152,7 +178,7 @@ struct UTF8 { template static bool Validate(InputStream& is, OutputStream& os) { #define COPY() os.Put(c = is.Take()) -#define TRANS(mask) result &= ((GetRange((unsigned char)c) & mask) != 0) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) Ch c; COPY(); @@ -160,7 +186,7 @@ struct UTF8 { return true; bool result = true; - switch (GetRange((unsigned char)c)) { + switch (GetRange(static_cast(c))) { case 2: TAIL(); return result; case 3: TAIL(); TAIL(); return result; case 4: COPY(); TRANS(0x50); TAIL(); return result; @@ -196,12 +222,12 @@ struct UTF8 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - Ch c = Take(is); - if ((unsigned char)c != 0xEFu) return c; + typename InputByteStream::Ch c = Take(is); + if (static_cast(c) != 0xEFu) return c; c = is.Take(); - if ((unsigned char)c != 0xBBu) return c; + if (static_cast(c) != 0xBBu) return c; c = is.Take(); - if ((unsigned char)c != 0xBFu) return c; + if (static_cast(c) != 0xBFu) return c; c = is.Take(); return c; } @@ -209,13 +235,15 @@ struct UTF8 { template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return is.Take(); + return static_cast(is.Take()); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xEFu); os.Put(0xBBu); os.Put(0xBFu); + os.Put(static_cast(0xEFu)); + os.Put(static_cast(0xBBu)); + os.Put(static_cast(0xBFu)); } template @@ -255,22 +283,38 @@ struct UTF16 { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; os.Put(static_cast((v >> 10) | 0xD800)); - os.Put((v & 0x3FF) | 0xDC00); + os.Put(static_cast((v & 0x3FF) | 0xDC00)); + } + } + + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + PutUnsafe(os, static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + PutUnsafe(os, static_cast((v >> 10) | 0xD800)); + PutUnsafe(os, static_cast((v & 0x3FF) | 0xDC00)); } } template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - Ch c = is.Take(); + typename InputStream::Ch c = is.Take(); if (c < 0xD800 || c > 0xDFFF) { - *codepoint = c; + *codepoint = static_cast(c); return true; } else if (c <= 0xDBFF) { - *codepoint = (c & 0x3FF) << 10; + *codepoint = (static_cast(c) & 0x3FF) << 10; c = is.Take(); - *codepoint |= (c & 0x3FF); + *codepoint |= (static_cast(c) & 0x3FF); *codepoint += 0x10000; return c >= 0xDC00 && c <= 0xDFFF; } @@ -281,8 +325,8 @@ struct UTF16 { static bool Validate(InputStream& is, OutputStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - Ch c; - os.Put(c = is.Take()); + typename InputStream::Ch c; + os.Put(static_cast(c = is.Take())); if (c < 0xD800 || c > 0xDFFF) return true; else if (c <= 0xDBFF) { @@ -300,28 +344,29 @@ struct UTF16LE : UTF16 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned short)c == 0xFEFFu ? Take(is) : c; + return static_cast(c) == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take(); - c |= (unsigned char)is.Take() << 8; - return c; + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFFu); os.Put(0xFEu); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(c & 0xFFu); - os.Put((c >> 8) & 0xFFu); + os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); } }; @@ -332,28 +377,29 @@ struct UTF16BE : UTF16 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned short)c == 0xFEFFu ? Take(is) : c; + return static_cast(c) == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take(); - return c; + unsigned c = static_cast(static_cast(is.Take())) << 8; + c |= static_cast(is.Take()); + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFEu); os.Put(0xFFu); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put((c >> 8) & 0xFFu); - os.Put(c & 0xFFu); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(static_cast(static_cast(c) & 0xFFu)); } }; @@ -382,6 +428,13 @@ struct UTF32 { os.Put(codepoint); } + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, codepoint); + } + template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); @@ -406,32 +459,35 @@ struct UTF32LE : UTF32 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned)c == 0x0000FEFFu ? Take(is) : c; + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take(); - c |= (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take() << 16; - c |= (unsigned char)is.Take() << 24; - return c; + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 24; + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0xFFu); os.Put(0xFEu); os.Put(0x00u); os.Put(0x00u); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(c & 0xFFu); - os.Put((c >> 8) & 0xFFu); - os.Put((c >> 16) & 0xFFu); - os.Put((c >> 24) & 0xFFu); + os.Put(static_cast(c & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 24) & 0xFFu)); } }; @@ -442,32 +498,35 @@ struct UTF32BE : UTF32 { static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); - return (unsigned)c == 0x0000FEFFu ? Take(is) : c; + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = (unsigned char)is.Take() << 24; - c |= (unsigned char)is.Take() << 16; - c |= (unsigned char)is.Take() << 8; - c |= (unsigned char)is.Take(); - return c; + unsigned c = static_cast(static_cast(is.Take())) << 24; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(0x00u); os.Put(0x00u); os.Put(0xFEu); os.Put(0xFFu); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put((c >> 24) & 0xFFu); - os.Put((c >> 16) & 0xFFu); - os.Put((c >> 8) & 0xFFu); - os.Put(c & 0xFFu); + os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast(c & 0xFFu)); } }; @@ -491,31 +550,37 @@ struct ASCII { os.Put(static_cast(codepoint & 0xFF)); } + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + PutUnsafe(os, static_cast(codepoint & 0xFF)); + } + template static bool Decode(InputStream& is, unsigned* codepoint) { - unsigned char c = static_cast(is.Take()); + uint8_t c = static_cast(is.Take()); *codepoint = c; return c <= 0X7F; } template static bool Validate(InputStream& is, OutputStream& os) { - unsigned char c = is.Take(); - os.Put(c); + uint8_t c = static_cast(is.Take()); + os.Put(static_cast(c)); return c <= 0x7F; } template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - Ch c = Take(is); - return c; + uint8_t c = static_cast(Take(is)); + return static_cast(c); } template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return is.Take(); + return static_cast(is.Take()); } template @@ -555,21 +620,28 @@ struct AutoUTF { #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x template - RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { + static RAPIDJSON_FORCEINLINE void Encode(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; (*f[os.GetType()])(os, codepoint); } + template + static RAPIDJSON_FORCEINLINE void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; + (*f[os.GetType()])(os, codepoint); + } + template - RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { + static RAPIDJSON_FORCEINLINE bool Decode(InputStream& is, unsigned* codepoint) { typedef bool (*DecodeFunc)(InputStream&, unsigned*); static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; return (*f[is.GetType()])(is, codepoint); } template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { typedef bool (*ValidateFunc)(InputStream&, OutputStream&); static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; return (*f[is.GetType()])(is, os); @@ -586,7 +658,7 @@ template struct Transcoder { //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; @@ -594,31 +666,50 @@ struct Transcoder { return true; } + template + static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::EncodeUnsafe(os, codepoint); + return true; + } + //! Validate one Unicode codepoint from an encoded stream. template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { return Transcode(is, os); // Since source/target encoding is different, must transcode. } }; +// Forward declaration. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c); + //! Specialization of Transcoder with same source and target encoding. template struct Transcoder { template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { return Encoding::Validate(is, os); // source/target encoding are the same } }; RAPIDJSON_NAMESPACE_END -#if defined(__GNUC__) || defined(_MSV_VER) +#if defined(__GNUC__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/error/en.h b/contrib/rapidjson/include/rapidjson/error/en.h index d5f9caab8..2db838bff 100644 --- a/contrib/rapidjson/include/rapidjson/error/en.h +++ b/contrib/rapidjson/include/rapidjson/error/en.h @@ -12,11 +12,17 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#ifndef RAPIDJSON_ERROR_EN_H__ -#define RAPIDJSON_ERROR_EN_H__ +#ifndef RAPIDJSON_ERROR_EN_H_ +#define RAPIDJSON_ERROR_EN_H_ #include "error.h" +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(covered-switch-default) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Maps error code of parsing into error message. @@ -32,7 +38,7 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); - case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); + case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); @@ -55,11 +61,14 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); - default: - return RAPIDJSON_ERROR_STRING("Unknown error."); + default: return RAPIDJSON_ERROR_STRING("Unknown error."); } } RAPIDJSON_NAMESPACE_END -#endif // RAPIDJSON_ERROR_EN_H__ +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_EN_H_ diff --git a/contrib/rapidjson/include/rapidjson/error/error.h b/contrib/rapidjson/include/rapidjson/error/error.h index f9094fb95..9311d2f03 100644 --- a/contrib/rapidjson/include/rapidjson/error/error.h +++ b/contrib/rapidjson/include/rapidjson/error/error.h @@ -12,11 +12,16 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#ifndef RAPIDJSON_ERROR_ERROR_H__ -#define RAPIDJSON_ERROR_ERROR_H__ +#ifndef RAPIDJSON_ERROR_ERROR_H_ +#define RAPIDJSON_ERROR_ERROR_H_ #include "../rapidjson.h" +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + /*! \file error.h */ /*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ @@ -99,7 +104,9 @@ enum ParseErrorCode { \see GenericReader::Parse, GenericDocument::Parse */ struct ParseResult { - + //!! Unspecified boolean type + typedef bool (ParseResult::*BooleanType)() const; +public: //! Default constructor, no error. ParseResult() : code_(kParseErrorNone), offset_(0) {} //! Constructor to set an error. @@ -110,8 +117,8 @@ struct ParseResult { //! Get the error offset, if \ref IsError(), 0 otherwise. size_t Offset() const { return offset_; } - //! Conversion to \c bool, returns \c true, iff !\ref IsError(). - operator bool() const { return !IsError(); } + //! Explicit conversion to \c bool, returns \c true, iff !\ref IsError(). + operator BooleanType() const { return !IsError() ? &ParseResult::IsError : NULL; } //! Whether the result is an error. bool IsError() const { return code_ != kParseErrorNone; } @@ -119,6 +126,10 @@ struct ParseResult { bool operator==(ParseErrorCode code) const { return code_ == code; } friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + bool operator!=(const ParseResult& that) const { return !(*this == that); } + bool operator!=(ParseErrorCode code) const { return !(*this == code); } + friend bool operator!=(ParseErrorCode code, const ParseResult & err) { return err != code; } + //! Reset error code. void Clear() { Set(kParseErrorNone); } //! Update error code and offset. @@ -143,4 +154,8 @@ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); RAPIDJSON_NAMESPACE_END -#endif // RAPIDJSON_ERROR_ERROR_H__ +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_ERROR_H_ diff --git a/contrib/rapidjson/include/rapidjson/filereadstream.h b/contrib/rapidjson/include/rapidjson/filereadstream.h index 3913eb74b..b56ea13b3 100644 --- a/contrib/rapidjson/include/rapidjson/filereadstream.h +++ b/contrib/rapidjson/include/rapidjson/filereadstream.h @@ -15,9 +15,16 @@ #ifndef RAPIDJSON_FILEREADSTREAM_H_ #define RAPIDJSON_FILEREADSTREAM_H_ -#include "rapidjson.h" +#include "stream.h" #include +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! File byte stream for input using fread(). @@ -85,4 +92,8 @@ private: RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_FILESTREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/filewritestream.h b/contrib/rapidjson/include/rapidjson/filewritestream.h index dfb9cbd02..6378dd60e 100644 --- a/contrib/rapidjson/include/rapidjson/filewritestream.h +++ b/contrib/rapidjson/include/rapidjson/filewritestream.h @@ -15,9 +15,14 @@ #ifndef RAPIDJSON_FILEWRITESTREAM_H_ #define RAPIDJSON_FILEWRITESTREAM_H_ -#include "rapidjson.h" +#include "stream.h" #include +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Wrapper of C file stream for input using fread(). @@ -57,7 +62,11 @@ public: void Flush() { if (current_ != buffer_) { - fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } current_ = buffer_; } } @@ -88,4 +97,8 @@ inline void PutN(FileWriteStream& stream, char c, size_t n) { RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_FILESTREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/fwd.h b/contrib/rapidjson/include/rapidjson/fwd.h new file mode 100644 index 000000000..e8104e841 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/fwd.h @@ -0,0 +1,151 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FWD_H_ +#define RAPIDJSON_FWD_H_ + +#include "rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN + +// encodings.h + +template struct UTF8; +template struct UTF16; +template struct UTF16BE; +template struct UTF16LE; +template struct UTF32; +template struct UTF32BE; +template struct UTF32LE; +template struct ASCII; +template struct AutoUTF; + +template +struct Transcoder; + +// allocators.h + +class CrtAllocator; + +template +class MemoryPoolAllocator; + +// stream.h + +template +struct GenericStringStream; + +typedef GenericStringStream > StringStream; + +template +struct GenericInsituStringStream; + +typedef GenericInsituStringStream > InsituStringStream; + +// stringbuffer.h + +template +class GenericStringBuffer; + +typedef GenericStringBuffer, CrtAllocator> StringBuffer; + +// filereadstream.h + +class FileReadStream; + +// filewritestream.h + +class FileWriteStream; + +// memorybuffer.h + +template +struct GenericMemoryBuffer; + +typedef GenericMemoryBuffer MemoryBuffer; + +// memorystream.h + +struct MemoryStream; + +// reader.h + +template +struct BaseReaderHandler; + +template +class GenericReader; + +typedef GenericReader, UTF8, CrtAllocator> Reader; + +// writer.h + +template +class Writer; + +// prettywriter.h + +template +class PrettyWriter; + +// document.h + +template +struct GenericMember; + +template +class GenericMemberIterator; + +template +struct GenericStringRef; + +template +class GenericValue; + +typedef GenericValue, MemoryPoolAllocator > Value; + +template +class GenericDocument; + +typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; + +// pointer.h + +template +class GenericPointer; + +typedef GenericPointer Pointer; + +// schema.h + +template +class IGenericRemoteSchemaDocumentProvider; + +template +class GenericSchemaDocument; + +typedef GenericSchemaDocument SchemaDocument; +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +template < + typename SchemaDocumentType, + typename OutputHandler, + typename StateAllocator> +class GenericSchemaValidator; + +typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/biginteger.h b/contrib/rapidjson/include/rapidjson/internal/biginteger.h index 99a30acf6..9d3e88c99 100644 --- a/contrib/rapidjson/include/rapidjson/internal/biginteger.h +++ b/contrib/rapidjson/include/rapidjson/internal/biginteger.h @@ -19,6 +19,7 @@ #if defined(_MSC_VER) && defined(_M_AMD64) #include // for _umul128 +#pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN @@ -50,7 +51,16 @@ public: if (length > 0) AppendDecimal64(decimals + i, decimals + i + length); } - + + BigInteger& operator=(const BigInteger &rhs) + { + if (this != &rhs) { + count_ = rhs.count_; + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + return *this; + } + BigInteger& operator=(uint64_t u) { digits_[0] = u; count_ = 1; @@ -230,7 +240,7 @@ private: uint64_t r = 0; for (const char* p = begin; p != end; ++p) { RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10 + (*p - '0'); + r = r * 10u + static_cast(*p - '0'); } return r; } diff --git a/contrib/rapidjson/include/rapidjson/internal/diyfp.h b/contrib/rapidjson/include/rapidjson/internal/diyfp.h index 3b6c4238c..29abf8046 100644 --- a/contrib/rapidjson/include/rapidjson/internal/diyfp.h +++ b/contrib/rapidjson/include/rapidjson/internal/diyfp.h @@ -21,9 +21,10 @@ #include "../rapidjson.h" -#if defined(_MSC_VER) && defined(_M_AMD64) +#if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) #include #pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN @@ -34,8 +35,13 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + struct DiyFp { - DiyFp() {} + DiyFp() : f(), e() {} DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} @@ -232,8 +238,8 @@ inline DiyFp GetCachedPower(int e, int* K) { } inline DiyFp GetCachedPower10(int exp, int *outExp) { - unsigned index = (exp + 348) / 8; - *outExp = -348 + index * 8; + unsigned index = (static_cast(exp) + 348u) / 8u; + *outExp = -348 + static_cast(index) * 8; return GetCachedPowerByIndex(index); } @@ -241,6 +247,11 @@ inline DiyFp GetCachedPower10(int exp, int *outExp) { RAPIDJSON_DIAG_POP #endif +#ifdef __clang__ +RAPIDJSON_DIAG_POP +RAPIDJSON_DIAG_OFF(padded) +#endif + } // namespace internal RAPIDJSON_NAMESPACE_END diff --git a/contrib/rapidjson/include/rapidjson/internal/dtoa.h b/contrib/rapidjson/include/rapidjson/internal/dtoa.h index 2d8d2e46a..bf2e9b2e5 100644 --- a/contrib/rapidjson/include/rapidjson/internal/dtoa.h +++ b/contrib/rapidjson/include/rapidjson/internal/dtoa.h @@ -29,6 +29,7 @@ namespace internal { #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) +RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 #endif inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { @@ -40,7 +41,7 @@ inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uin } } -inline unsigned CountDecimalDigit32(uint32_t n) { +inline int CountDecimalDigit32(uint32_t n) { // Simple pure C++ implementation was faster than __builtin_clz version in this situation. if (n < 10) return 1; if (n < 100) return 2; @@ -101,7 +102,8 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff kappa--; if (p2 < delta) { *K += kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * kPow10[-kappa]); + int index = -kappa; + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); return; } } @@ -145,10 +147,10 @@ inline char* WriteExponent(int K, char* buffer) { return buffer; } -inline char* Prettify(char* buffer, int length, int k) { +inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { const int kk = length + k; // 10^(kk-1) <= v < 10^kk - if (length <= kk && kk <= 21) { + if (0 <= k && kk <= 21) { // 1234e7 -> 12340000000 for (int i = length; i < kk; i++) buffer[i] = '0'; @@ -158,19 +160,44 @@ inline char* Prettify(char* buffer, int length, int k) { } else if (0 < kk && kk <= 21) { // 1234e-2 -> 12.34 - std::memmove(&buffer[kk + 1], &buffer[kk], length - kk); + std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); buffer[kk] = '.'; - return &buffer[length + 1]; + if (0 > k + maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[kk + 2]; // Reserve one zero + } + else + return &buffer[length + 1]; } else if (-6 < kk && kk <= 0) { // 1234e-6 -> 0.001234 const int offset = 2 - kk; - std::memmove(&buffer[offset], &buffer[0], length); + std::memmove(&buffer[offset], &buffer[0], static_cast(length)); buffer[0] = '0'; buffer[1] = '.'; for (int i = 2; i < offset; i++) buffer[i] = '0'; - return &buffer[length + offset]; + if (length - kk > maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = maxDecimalPlaces + 1; i > 2; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[3]; // Reserve one zero + } + else + return &buffer[length + offset]; + } + else if (kk < -maxDecimalPlaces) { + // Truncate to zero + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; } else if (length == 1) { // 1e30 @@ -179,14 +206,15 @@ inline char* Prettify(char* buffer, int length, int k) { } else { // 1234e30 -> 1.234e33 - std::memmove(&buffer[2], &buffer[1], length - 1); + std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); buffer[1] = '.'; buffer[length + 1] = 'e'; return WriteExponent(kk - 1, &buffer[0 + length + 2]); } } -inline char* dtoa(double value, char* buffer) { +inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { + RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); Double d(value); if (d.IsZero()) { if (d.Sign()) @@ -203,7 +231,7 @@ inline char* dtoa(double value, char* buffer) { } int length, K; Grisu2(value, buffer, &length, &K); - return Prettify(buffer, length, K); + return Prettify(buffer, length, K, maxDecimalPlaces); } } diff --git a/contrib/rapidjson/include/rapidjson/internal/ieee754.h b/contrib/rapidjson/include/rapidjson/internal/ieee754.h index e3f03364c..c2684ba2a 100644 --- a/contrib/rapidjson/include/rapidjson/internal/ieee754.h +++ b/contrib/rapidjson/include/rapidjson/internal/ieee754.h @@ -40,6 +40,7 @@ public: bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } + bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } @@ -47,7 +48,7 @@ public: int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } - static unsigned EffectiveSignificandSize(int order) { + static int EffectiveSignificandSize(int order) { if (order >= -1021) return 53; else if (order <= -1074) diff --git a/contrib/rapidjson/include/rapidjson/internal/regex.h b/contrib/rapidjson/include/rapidjson/internal/regex.h new file mode 100644 index 000000000..e1a2faae5 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/internal/regex.h @@ -0,0 +1,734 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_REGEX_H_ +#define RAPIDJSON_INTERNAL_REGEX_H_ + +#include "../allocators.h" +#include "../stream.h" +#include "stack.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 7 +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +#ifndef RAPIDJSON_REGEX_VERBOSE +#define RAPIDJSON_REGEX_VERBOSE 0 +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// DecodedStream + +template +class DecodedStream { +public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + +private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericRegex + +static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 +static const SizeType kRegexInvalidRange = ~SizeType(0); + +template +class GenericRegexSearch; + +//! Regular expression engine with subset of ECMAscript grammar. +/*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html +*/ +template +class GenericRegex { +public: + typedef Encoding EncodingType; + typedef typename Encoding::Ch Ch; + template friend class GenericRegexSearch; + + GenericRegex(const Ch* source, Allocator* allocator = 0) : + states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + anchorBegin_(), anchorEnd_() + { + GenericStringStream ss(source); + DecodedStream, Encoding> ds(ss); + Parse(ds); + } + + ~GenericRegex() {} + + bool IsValid() const { + return root_ != kRegexInvalidState; + } + +private: + enum Operator { + kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis + }; + + static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' + static const unsigned kRangeCharacterClass = 0xFFFFFFFE; + static const unsigned kRangeNegationFlag = 0x80000000; + + struct Range { + unsigned start; // + unsigned end; + SizeType next; + }; + + struct State { + SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; + }; + + struct Frag { + Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; + }; + + State& GetState(SizeType index) { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + const State& GetState(SizeType index) const { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + Range& GetRange(SizeType index) { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + const Range& GetRange(SizeType index) const { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + template + void Parse(DecodedStream& ds) { + Allocator allocator; + Stack operandStack(&allocator, 256); // Frag + Stack operatorStack(&allocator, 256); // Operator + Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) { + switch (codepoint = ds.Take()) { + case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { + unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') { + ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; + } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { + SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); + } + } + + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) { + Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; + +#if RAPIDJSON_REGEX_VERBOSE + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) { + State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); + } + printf("\n"); +#endif + } + } + + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { + State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; + } + + void PushOperand(Stack& operandStack, unsigned codepoint) { + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); + } + + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { + if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; + } + + SizeType Append(SizeType l1, SizeType l2) { + SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; + } + + void Patch(SizeType l, SizeType s) { + for (SizeType next; l != kRegexInvalidState; l = next) { + next = GetState(l).out; + GetState(l).out = s; + } + } + + bool Eval(Stack& operandStack, Operator op) { + switch (op) { + case kConcatenation: + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); + return true; + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + default: + RAPIDJSON_ASSERT(op == kOneOrMore); + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + } + } + + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { + RAPIDJSON_ASSERT(n <= m); + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) { + if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else { + Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? + } + return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) { + CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; + } + + static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } + + void CloneTopOperand(Stack& operandStack) { + const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) { + if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; + } + + template + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') { + if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; + } + + template + bool ParseRange(DecodedStream& ds, SizeType* range) { + bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) { + if (isBegin) { + isBegin = false; + if (codepoint == '^') { + negate = true; + continue; + } + } + + switch (codepoint) { + case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) { // Add trailing '-' + SizeType r = NewRange('-'); + RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; + } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; + + case '\\': + if (ds.Peek() == 'b') { + ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) { + case 1: + if (codepoint == '-') { + step++; + break; + } + // fall through to step 0 for other characters + + case 0: + { + SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; + } + } + } + return false; + } + + SizeType NewRange(unsigned codepoint) { + Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; + } + + template + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + unsigned codepoint; + switch (codepoint = ds.Take()) { + case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } + } + + Stack states_; + Stack ranges_; + SizeType root_; + SizeType stateCount_; + SizeType rangeCount_; + + static const unsigned kInfinityQuantifier = ~0u; + + // For SearchWithAnchoring() + bool anchorBegin_; + bool anchorEnd_; +}; + +template +class GenericRegexSearch { +public: + typedef typename RegexType::EncodingType Encoding; + typedef typename Encoding::Ch Ch; + + GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : + regex_(regex), allocator_(allocator), ownAllocator_(0), + state0_(allocator, 0), state1_(allocator, 0), stateSet_() + { + RAPIDJSON_ASSERT(regex_.IsValid()); + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); + state0_.template Reserve(regex_.stateCount_); + state1_.template Reserve(regex_.stateCount_); + } + + ~GenericRegexSearch() { + Allocator::Free(stateSet_); + RAPIDJSON_DELETE(ownAllocator_); + } + + template + bool Match(InputStream& is) { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) { + return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); + } + + bool Search(const Ch* s) { + GenericStringStream is(s); + return Search(is); + } + +private: + typedef typename RegexType::State State; + typedef typename RegexType::Range Range; + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, regex_.root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = regex_.GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == RegexType::kAnyCharacterClass || + (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, regex_.root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (regex_.stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) { + RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = regex_.GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { + stateSet_[index >> 5] |= (1u << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = regex_.GetRange(rangeIndex); + if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + const RegexType& regex_; + Allocator* allocator_; + Allocator* ownAllocator_; + Stack state0_; + Stack state1_; + uint32_t* stateSet_; +}; + +typedef GenericRegex > Regex; +typedef GenericRegexSearch RegexSearch; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/stack.h b/contrib/rapidjson/include/rapidjson/internal/stack.h index 722d56923..5c5398c35 100644 --- a/contrib/rapidjson/include/rapidjson/internal/stack.h +++ b/contrib/rapidjson/include/rapidjson/internal/stack.h @@ -15,7 +15,13 @@ #ifndef RAPIDJSON_INTERNAL_STACK_H_ #define RAPIDJSON_INTERNAL_STACK_H_ -#include "../rapidjson.h" +#include "../allocators.h" +#include "swap.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -32,7 +38,6 @@ public: // Optimization note: Do not allocate memory for stack_ in constructor. // Do it lazily when first Push() -> Expand() -> Resize(). Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { - RAPIDJSON_ASSERT(stackCapacity > 0); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -81,6 +86,15 @@ public: } #endif + void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(stack_, rhs.stack_); + internal::Swap(stackTop_, rhs.stackTop_); + internal::Swap(stackEnd_, rhs.stackEnd_); + internal::Swap(initialCapacity_, rhs.initialCapacity_); + } + void Clear() { stackTop_ = stack_; } void ShrinkToFit() { @@ -98,11 +112,22 @@ public: // Optimization note: try to minimize the size of this function for force inline. // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. template - RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { // Expand the stack if needed - if (stackTop_ + sizeof(T) * count >= stackEnd_) + if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) Expand(count); + } + template + RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + Reserve(count); + return PushUnsafe(count); + } + + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { + RAPIDJSON_ASSERT(stackTop_); + RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; @@ -122,9 +147,32 @@ public: } template - T* Bottom() { return (T*)stack_; } + const T* Top() const { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* End() { return reinterpret_cast(stackTop_); } + + template + const T* End() const { return reinterpret_cast(stackTop_); } + + template + T* Bottom() { return reinterpret_cast(stack_); } + + template + const T* Bottom() const { return reinterpret_cast(stack_); } + + bool HasAllocator() const { + return allocator_ != 0; + } + + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } - Allocator& GetAllocator() { return *allocator_; } bool Empty() const { return stackTop_ == stack_; } size_t GetSize() const { return static_cast(stackTop_ - stack_); } size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } @@ -136,7 +184,7 @@ private: size_t newCapacity; if (stack_ == 0) { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); newCapacity = initialCapacity_; } else { newCapacity = GetCapacity(); @@ -151,7 +199,7 @@ private: void Resize(size_t newCapacity) { const size_t size = GetSize(); // Backup the current size - stack_ = (char*)allocator_->Realloc(stack_, GetCapacity(), newCapacity); + stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); stackTop_ = stack_ + size; stackEnd_ = stack_ + newCapacity; } @@ -176,4 +224,8 @@ private: } // namespace internal RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_STACK_H_ diff --git a/contrib/rapidjson/include/rapidjson/internal/strfunc.h b/contrib/rapidjson/include/rapidjson/internal/strfunc.h index 84405065a..226439a76 100644 --- a/contrib/rapidjson/include/rapidjson/internal/strfunc.h +++ b/contrib/rapidjson/include/rapidjson/internal/strfunc.h @@ -15,7 +15,8 @@ #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ #define RAPIDJSON_INTERNAL_STRFUNC_H_ -#include "../rapidjson.h" +#include "../stream.h" +#include RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -28,11 +29,40 @@ namespace internal { */ template inline SizeType StrLen(const Ch* s) { + RAPIDJSON_ASSERT(s != 0); const Ch* p = s; while (*p) ++p; return SizeType(p - s); } +template <> +inline SizeType StrLen(const char* s) { + return SizeType(std::strlen(s)); +} + +template <> +inline SizeType StrLen(const wchar_t* s) { + return SizeType(std::wcslen(s)); +} + +//! Returns number of code points in a encoded string. +template +bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { + RAPIDJSON_ASSERT(s != 0); + RAPIDJSON_ASSERT(outCount != 0); + GenericStringStream is(s); + const typename Encoding::Ch* end = s + length; + SizeType count = 0; + while (is.src_ < end) { + unsigned codepoint; + if (!Encoding::Decode(is, &codepoint)) + return false; + count++; + } + *outCount = count; + return true; +} + } // namespace internal RAPIDJSON_NAMESPACE_END diff --git a/contrib/rapidjson/include/rapidjson/internal/strtod.h b/contrib/rapidjson/include/rapidjson/internal/strtod.h index ace65f677..adf49e349 100644 --- a/contrib/rapidjson/include/rapidjson/internal/strtod.h +++ b/contrib/rapidjson/include/rapidjson/internal/strtod.h @@ -15,7 +15,6 @@ #ifndef RAPIDJSON_STRTOD_ #define RAPIDJSON_STRTOD_ -#include "../rapidjson.h" #include "ieee754.h" #include "biginteger.h" #include "diyfp.h" @@ -95,13 +94,13 @@ inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { hS_Exp2 -= common_Exp2; BigInteger dS = d; - dS.MultiplyPow5(dS_Exp5) <<= dS_Exp2; + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); BigInteger bS(bInt); - bS.MultiplyPow5(bS_Exp5) <<= bS_Exp2; + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); BigInteger hS(1); - hS.MultiplyPow5(hS_Exp5) <<= hS_Exp2; + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); BigInteger delta(0); dS.Difference(bS, &delta); @@ -134,22 +133,22 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) break; - significand = significand * 10 + (decimals[i] - '0'); + significand = significand * 10u + static_cast(decimals[i] - '0'); } if (i < length && decimals[i] >= '5') // Rounding significand++; size_t remaining = length - i; - const unsigned kUlpShift = 3; - const unsigned kUlp = 1 << kUlpShift; - int error = (remaining == 0) ? 0 : kUlp / 2; + const int kUlpShift = 3; + const int kUlp = 1 << kUlpShift; + int64_t error = (remaining == 0) ? 0 : kUlp / 2; DiyFp v(significand, 0); v = v.Normalize(); error <<= -v.e; - const int dExp = (int)decimalPosition - (int)i + exp; + const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; int actualExp; DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); @@ -163,10 +162,10 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 }; - int adjustment = dExp - actualExp - 1; + int adjustment = dExp - actualExp - 1; RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); v = v * kPow10[adjustment]; - if (length + adjustment > 19) // has more digits than decimal digits in 64-bit + if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit error += kUlp / 2; } @@ -178,10 +177,10 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit v = v.Normalize(); error <<= oldExp - v.e; - const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); - unsigned precisionSize = 64 - effectiveSignificandSize; + const int effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + int precisionSize = 64 - effectiveSignificandSize; if (precisionSize + kUlpShift >= 64) { - unsigned scaleExp = (precisionSize + kUlpShift) - 63; + int scaleExp = (precisionSize + kUlpShift) - 63; v.f >>= scaleExp; v.e += scaleExp; error = (error >> scaleExp) + 1 + kUlp; @@ -191,7 +190,7 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit DiyFp rounded(v.f >> precisionSize, v.e + precisionSize); const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; - if (precisionBits >= halfWay + error) { + if (precisionBits >= halfWay + static_cast(error)) { rounded.f++; if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) rounded.f >>= 1; @@ -201,12 +200,12 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit *result = rounded.ToDouble(); - return halfWay - error >= precisionBits || precisionBits >= halfWay + error; + return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { const BigInteger dInt(decimals, length); - const int dExp = (int)decimalPosition - (int)length + exp; + const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; Double a(approx); int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); if (cmp < 0) @@ -246,10 +245,10 @@ inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t // Trim right-most digits const int kMaxDecimalDigit = 780; - if ((int)length > kMaxDecimalDigit) { - int delta = (int(length) - kMaxDecimalDigit); + if (static_cast(length) > kMaxDecimalDigit) { + int delta = (static_cast(length) - kMaxDecimalDigit); exp += delta; - decimalPosition -= delta; + decimalPosition -= static_cast(delta); length = kMaxDecimalDigit; } diff --git a/contrib/rapidjson/include/rapidjson/internal/swap.h b/contrib/rapidjson/include/rapidjson/internal/swap.h index 0590921f1..666e49f97 100644 --- a/contrib/rapidjson/include/rapidjson/internal/swap.h +++ b/contrib/rapidjson/include/rapidjson/internal/swap.h @@ -17,6 +17,11 @@ #include "../rapidjson.h" +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -34,4 +39,8 @@ inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { } // namespace internal RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/contrib/rapidjson/include/rapidjson/istreamwrapper.h b/contrib/rapidjson/include/rapidjson/istreamwrapper.h new file mode 100644 index 000000000..8639c8c3c --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/istreamwrapper.h @@ -0,0 +1,115 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ISTREAMWRAPPER_H_ +#define RAPIDJSON_ISTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::istringstream + - \c std::stringstream + - \c std::wistringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wifstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_istream. +*/ + +template +class BasicIStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} + + Ch Peek() const { + typename StreamType::int_type c = stream_.peek(); + return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : static_cast('\0'); + } + + Ch Take() { + typename StreamType::int_type c = stream_.get(); + if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { + count_++; + return static_cast(c); + } + else + return '\0'; + } + + // tellg() may return -1 when failed. So we count by ourself. + size_t Tell() const { return count_; } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. + int i; + bool hasError = false; + for (i = 0; i < 4; ++i) { + typename StreamType::int_type c = stream_.get(); + if (c == StreamType::traits_type::eof()) { + hasError = true; + stream_.clear(); + break; + } + peekBuffer_[i] = static_cast(c); + } + for (--i; i >= 0; --i) + stream_.putback(peekBuffer_[i]); + return !hasError ? peekBuffer_ : 0; + } + +private: + BasicIStreamWrapper(const BasicIStreamWrapper&); + BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); + + StreamType& stream_; + size_t count_; //!< Number of characters read. Note: + mutable Ch peekBuffer_[4]; +}; + +typedef BasicIStreamWrapper IStreamWrapper; +typedef BasicIStreamWrapper WIStreamWrapper; + +#if defined(__clang__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/contrib/rapidjson/include/rapidjson/memorybuffer.h b/contrib/rapidjson/include/rapidjson/memorybuffer.h index 2484b2185..39bee1dec 100644 --- a/contrib/rapidjson/include/rapidjson/memorybuffer.h +++ b/contrib/rapidjson/include/rapidjson/memorybuffer.h @@ -15,7 +15,7 @@ #ifndef RAPIDJSON_MEMORYBUFFER_H_ #define RAPIDJSON_MEMORYBUFFER_H_ -#include "rapidjson.h" +#include "stream.h" #include "internal/stack.h" RAPIDJSON_NAMESPACE_BEGIN diff --git a/contrib/rapidjson/include/rapidjson/memorystream.h b/contrib/rapidjson/include/rapidjson/memorystream.h index 99feae5d7..1d71d8a4f 100644 --- a/contrib/rapidjson/include/rapidjson/memorystream.h +++ b/contrib/rapidjson/include/rapidjson/memorystream.h @@ -15,7 +15,13 @@ #ifndef RAPIDJSON_MEMORYSTREAM_H_ #define RAPIDJSON_MEMORYSTREAM_H_ -#include "rapidjson.h" +#include "stream.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif RAPIDJSON_NAMESPACE_BEGIN @@ -36,8 +42,8 @@ struct MemoryStream { MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} - Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } - Ch Take() { return (src_ == end_) ? '\0' : *src_++; } + Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } + Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } size_t Tell() const { return static_cast(src_ - begin_); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } @@ -58,4 +64,8 @@ struct MemoryStream { RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h b/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h index a26fff4bf..3d4477b9a 100644 --- a/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h +++ b/contrib/rapidjson/include/rapidjson/msinttypes/stdint.h @@ -89,14 +89,14 @@ #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we should wrap include with 'extern "C++" {}' -// or compiler give many errors like this: +// compiling for ARM we have to wrap include with 'extern "C++" {}' +// or compiler would give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#ifdef __cplusplus +#if defined(__cplusplus) && !defined(_M_ARM) extern "C" { #endif # include -#ifdef __cplusplus +#if defined(__cplusplus) && !defined(_M_ARM) } #endif diff --git a/contrib/rapidjson/include/rapidjson/ostreamwrapper.h b/contrib/rapidjson/include/rapidjson/ostreamwrapper.h new file mode 100644 index 000000000..6f4667c08 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/ostreamwrapper.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ +#define RAPIDJSON_OSTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::ostringstream + - \c std::stringstream + - \c std::wpstringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wofstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_ostream. +*/ + +template +class BasicOStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + + void Put(Ch c) { + stream_.put(c); + } + + void Flush() { + stream_.flush(); + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + + StreamType& stream_; +}; + +typedef BasicOStreamWrapper OStreamWrapper; +typedef BasicOStreamWrapper WOStreamWrapper; + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/contrib/rapidjson/include/rapidjson/pointer.h b/contrib/rapidjson/include/rapidjson/pointer.h index 5d2aa8d63..0f377efec 100644 --- a/contrib/rapidjson/include/rapidjson/pointer.h +++ b/contrib/rapidjson/include/rapidjson/pointer.h @@ -18,6 +18,16 @@ #include "document.h" #include "internal/itoa.h" +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + RAPIDJSON_NAMESPACE_BEGIN static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token @@ -71,7 +81,7 @@ template class GenericPointer { public: typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value - typedef typename EncodingType::Ch Ch; //!< Character type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value //! A token is the basic units of internal representation. /*! @@ -96,7 +106,7 @@ public: //@{ //! Default constructor. - GenericPointer() : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Constructor that parses a string or URI fragment representation. /*! @@ -155,7 +165,7 @@ public: GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Copy constructor. - GenericPointer(const GenericPointer& rhs) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { *this = rhs; } @@ -230,7 +240,7 @@ public: template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) Append(T* name, Allocator* allocator = 0) const { - return Append(name, StrLen(name), allocator); + return Append(name, internal::StrLen(name), allocator); } #if RAPIDJSON_HAS_STDSTRING @@ -253,17 +263,18 @@ public: */ GenericPointer Append(SizeType index, Allocator* allocator = 0) const { char buffer[21]; - SizeType length = (sizeof(SizeType) == 4 ? internal::u32toa(index, buffer): internal::u64toa(index, buffer)) - buffer; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); buffer[length] = '\0'; if (sizeof(Ch) == 1) { - Token token = { (Ch*)buffer, length, index }; + Token token = { reinterpret_cast(buffer), length, index }; return Append(token, allocator); } else { Ch name[21]; for (size_t i = 0; i <= length; i++) - name[i] = buffer[i]; + name[i] = static_cast(buffer[i]); Token token = { name, length, index }; return Append(token, allocator); } @@ -271,7 +282,7 @@ public: //! Append a token by value, and return a new Pointer /*! - \param value Value (either Uint or String) to be appended. + \param token token to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ @@ -299,6 +310,9 @@ public: //@} + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } + //!@name Tokens //@{ @@ -390,7 +404,7 @@ public: bool exist = true; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { if (v->IsArray() && t->name[0] == '-' && t->length == 1) { - v->PushBack(Value().Move(), allocator); + v->PushBack(ValueType().Move(), allocator); v = &((*v)[v->Size() - 1]); exist = false; } @@ -408,7 +422,7 @@ public: if (t->index >= v->Size()) { v->Reserve(t->index + 1, allocator); while (t->index >= v->Size()) - v->PushBack(Value().Move(), allocator); + v->PushBack(ValueType().Move(), allocator); exist = false; } v = &((*v)[t->index]); @@ -416,7 +430,7 @@ public: else { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) { - v->AddMember(Value(t->name, t->length, allocator).Move(), Value().Move(), allocator); + v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end exist = false; } @@ -435,7 +449,6 @@ public: //! Creates a value in a document. /*! \param document A document to be resolved. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. \param alreadyExist If non-null, it stores whether the resolved value is already exist. \return The resolved newly created, or already exists value. */ @@ -452,9 +465,18 @@ public: //! Query a value in a subtree. /*! \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. */ - ValueType* Get(ValueType& root) const { + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { RAPIDJSON_ASSERT(IsValid()); ValueType* v = &root; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { @@ -463,18 +485,23 @@ public: { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) - return 0; + break; v = &m->value; } - break; + continue; case kArrayType: if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - return 0; + break; v = &((*v)[t->index]); - break; + continue; default: - return 0; + break; } + + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; } return v; } @@ -484,7 +511,9 @@ public: \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \return Pointer to the value if it can be resolved. Otherwise null. */ - const ValueType* Get(const ValueType& root) const { return Get(const_cast(root)); } + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { + return Get(const_cast(root), unresolvedTokenIndex); + } //@} @@ -525,7 +554,7 @@ public: //! Query a value in a subtree with default primitive value. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -555,7 +584,7 @@ public: //! Query a value in a document with default primitive value. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -601,7 +630,7 @@ public: //! Set a primitive value in a subtree. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -637,7 +666,7 @@ public: //! Set a primitive value in a document. /*! - \tparam T \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) @@ -729,7 +758,7 @@ private: */ Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) @@ -738,8 +767,12 @@ private: tokenCount_ = rhs.tokenCount_ + extraToken; tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); - std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + if (rhs.tokenCount_ > 0) { + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + } + if (nameBufferSize > 0) { + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + } // Adjust pointers to name buffer std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; @@ -759,11 +792,13 @@ private: } //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation /*! \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. \param length Length of the source string. \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. */ +#endif void Parse(const Ch* source, size_t length) { RAPIDJSON_ASSERT(source != NULL); RAPIDJSON_ASSERT(nameBuffer_ == 0); @@ -771,7 +806,7 @@ private: // Create own allocator if user did not supply. if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); // Count number of '/' as tokenCount tokenCount_ = 0; @@ -857,7 +892,7 @@ private: *name++ = c; } - token->length = name - token->name; + token->length = static_cast(name - token->name); if (token->length == 0) isNumber = false; *name++ = '\0'; // Null terminator @@ -944,6 +979,8 @@ private: */ class PercentDecodeStream { public: + typedef typename ValueType::Ch Ch; + //! Constructor /*! \param source Start of the stream @@ -959,11 +996,11 @@ private: src_++; Ch c = 0; for (int j = 0; j < 2; j++) { - c <<= 4; + c = static_cast(c << 4); Ch h = *src_; - if (h >= '0' && h <= '9') c += h - '0'; - else if (h >= 'A' && h <= 'F') c += h - 'A' + 10; - else if (h >= 'a' && h <= 'f') c += h - 'a' + 10; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); else { valid_ = false; return 0; @@ -973,7 +1010,7 @@ private: return c; } - size_t Tell() const { return src_ - head_; } + size_t Tell() const { return static_cast(src_ - head_); } bool IsValid() const { return valid_; } private: @@ -992,8 +1029,8 @@ private: unsigned char u = static_cast(c); static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; os_.Put('%'); - os_.Put(hexDigits[u >> 4]); - os_.Put(hexDigits[u & 15]); + os_.Put(static_cast(hexDigits[u >> 4])); + os_.Put(static_cast(hexDigits[u & 15])); } private: OutputStream& os_; @@ -1041,23 +1078,23 @@ typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, c ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer) { - return pointer.Get(root); +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); } template -const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer) { - return pointer.Get(root); +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); } template -typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N]) { - return GenericPointer(source, N - 1).Get(root); +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } template -const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N]) { - return GenericPointer(source, N - 1).Get(root); +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } ////////////////////////////////////////////////////////////////////////////// @@ -1310,4 +1347,12 @@ bool EraseValueByPointer(T& root, const CharType(&source)[N]) { RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_POINTER_H_ diff --git a/contrib/rapidjson/include/rapidjson/prettywriter.h b/contrib/rapidjson/include/rapidjson/prettywriter.h index 416dd492e..98dfb3060 100644 --- a/contrib/rapidjson/include/rapidjson/prettywriter.h +++ b/contrib/rapidjson/include/rapidjson/prettywriter.h @@ -22,8 +22,21 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN +//! Combination of PrettyWriter format flags. +/*! \see PrettyWriter::SetFormatOptions + */ +enum PrettyFormatOptions { + kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. +}; + //! Writer with indentation and spacing. /*! \tparam OutputStream Type of ouptut os. @@ -31,10 +44,10 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam TargetEncoding Encoding of output stream. \tparam StackAllocator Type of allocator for allocating memory of stack. */ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> -class PrettyWriter : public Writer { +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class PrettyWriter : public Writer { public: - typedef Writer Base; + typedef Writer Base; typedef typename Base::Ch Ch; //! Constructor @@ -42,8 +55,17 @@ public: \param allocator User supplied allocator. If it is null, it will create a private one. \param levelDepth Initial capacity of stack. */ - PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + PrettyWriter(PrettyWriter&& rhs) : + Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} +#endif //! Set custom indentation. /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). @@ -57,6 +79,14 @@ public: return *this; } + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { + formatOptions_ = options; + return *this; + } + /*! @name Implementation of Handler \see Handler */ @@ -70,7 +100,15 @@ public: bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); + (void)copy; + PrettyPrefix(kNumberType); + return Base::WriteString(str, length); + } + bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; PrettyPrefix(kStringType); return Base::WriteString(str, length); @@ -89,11 +127,19 @@ public: } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) { + return Key(str.data(), SizeType(str.size())); + } +#endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; if (!empty) { @@ -104,7 +150,7 @@ public: (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); + Base::Flush(); return true; } @@ -120,7 +166,7 @@ public: RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - if (!empty) { + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); WriteIndent(); } @@ -128,7 +174,7 @@ public: (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); + Base::Flush(); return true; } @@ -142,6 +188,22 @@ public: bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + PrettyPrefix(type); + return Base::WriteRawValue(json, length); + } + protected: void PrettyPrefix(Type type) { (void)type; @@ -151,11 +213,14 @@ protected: if (level->inArray) { if (level->valueCount > 0) { Base::os_->Put(','); // add comma if it is not the first element in array - Base::os_->Put('\n'); + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); } - else + + if (!(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); - WriteIndent(); + WriteIndent(); + } } else { // in object if (level->valueCount > 0) { @@ -186,11 +251,12 @@ protected: void WriteIndent() { size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, indentChar_, count); + PutN(*Base::os_, static_cast(indentChar_), count); } Ch indentChar_; unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; private: // Prohibit copy constructor & assignment operator. @@ -200,6 +266,10 @@ private: RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/rapidjson.h b/contrib/rapidjson/include/rapidjson/rapidjson.h index f22130d3c..5716fdc06 100644 --- a/contrib/rapidjson/include/rapidjson/rapidjson.h +++ b/contrib/rapidjson/include/rapidjson/rapidjson.h @@ -49,6 +49,11 @@ // token stringification #define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) #define RAPIDJSON_DO_STRINGIFY(x) #x + +// token concatenation +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y //!@endcond /*! \def RAPIDJSON_MAJOR_VERSION @@ -68,8 +73,8 @@ \brief Version of RapidJSON in ".." string format. */ #define RAPIDJSON_MAJOR_VERSION 1 -#define RAPIDJSON_MINOR_VERSION 0 -#define RAPIDJSON_PATCH_VERSION 2 +#define RAPIDJSON_MINOR_VERSION 1 +#define RAPIDJSON_PATCH_VERSION 0 #define RAPIDJSON_VERSION_STRING \ RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) @@ -119,6 +124,31 @@ #define RAPIDJSON_NAMESPACE_END } #endif +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_HAS_STDSTRING +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def RAPIDJSON_HAS_STDSTRING + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#endif // !defined(RAPIDJSON_HAS_STDSTRING) + +#if RAPIDJSON_HAS_STDSTRING +#include +#endif // RAPIDJSON_HAS_STDSTRING + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE @@ -134,7 +164,7 @@ */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#ifdef _MSC_VER +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else @@ -153,9 +183,9 @@ #ifndef RAPIDJSON_FORCEINLINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && !defined(NDEBUG) +#if defined(_MSC_VER) && defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __forceinline -#elif defined(__GNUC__) && __GNUC__ >= 4 && !defined(NDEBUG) +#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) #else #define RAPIDJSON_FORCEINLINE @@ -211,6 +241,8 @@ # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else @@ -223,7 +255,7 @@ //! Whether using 64-bit architecture #ifndef RAPIDJSON_64BIT -#if defined(__LP64__) || defined(_WIN64) +#if defined(__LP64__) || (defined(__x86_64__) && defined(__ILP32__)) || defined(_WIN64) || defined(__EMSCRIPTEN__) #define RAPIDJSON_64BIT 1 #else #define RAPIDJSON_64BIT 0 @@ -238,13 +270,14 @@ \param x pointer to align Some machines require strict data alignment. Currently the default uses 4 bytes - alignment. User can customize by defining the RAPIDJSON_ALIGN function macro., + alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. + User can customize by defining the RAPIDJSON_ALIGN function macro. */ #ifndef RAPIDJSON_ALIGN #if RAPIDJSON_64BIT == 1 -#define RAPIDJSON_ALIGN(x) ((x + 7u) & ~7u) +#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) #else -#define RAPIDJSON_ALIGN(x) ((x + 3u) & ~3u) +#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) #endif #endif @@ -262,17 +295,47 @@ #endif /////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD +// RAPIDJSON_48BITPOINTER_OPTIMIZATION + +//! Use only lower 48-bit address for some pointers. +/*! + \ingroup RAPIDJSON_CONFIG + + This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. + The higher 16-bit can be used for storing other data. + \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. +*/ +#ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 +#else +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 +#endif +#endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION + +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 +#if RAPIDJSON_64BIT != 1 +#error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 +#endif +#define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) +#define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) +#else +#define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) +#define RAPIDJSON_GETPOINTER(type, p) (p) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_NEON/RAPIDJSON_SIMD /*! \def RAPIDJSON_SIMD \ingroup RAPIDJSON_CONFIG - \brief Enable SSE2/SSE4.2 optimization. + \brief Enable SSE2/SSE4.2/Neon optimization. RapidJSON supports optimized implementations for some parsing operations - based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible - processors. + based on the SSE2, SSE4.2 or NEon SIMD extensions on modern Intel + or ARM compatible processors. - To enable these optimizations, two different symbols can be defined; + To enable these optimizations, three different symbols can be defined; \code // Enable SSE2 optimization. #define RAPIDJSON_SSE2 @@ -281,13 +344,17 @@ #define RAPIDJSON_SSE42 \endcode - \c RAPIDJSON_SSE42 takes precedence, if both are defined. + // Enable ARM Neon optimization. + #define RAPIDJSON_NEON + \endcode + + \c RAPIDJSON_SSE42 takes precedence over SSE2, if both are defined. If any of these symbols is defined, RapidJSON defines the macro \c RAPIDJSON_SIMD to indicate the availability of the optimized code. */ #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ - || defined(RAPIDJSON_DOXYGEN_RUNNING) + || defined(RAPIDJSON_NEON) || defined(RAPIDJSON_DOXYGEN_RUNNING) #define RAPIDJSON_SIMD #endif @@ -347,25 +414,33 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_STATIC_ASSERT -// Adopt from boost +// Prefer C++11 static_assert, if available #ifndef RAPIDJSON_STATIC_ASSERT +#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#define RAPIDJSON_STATIC_ASSERT(x) \ + static_assert(x, RAPIDJSON_STRINGIFY(x)) +#endif // C++11 +#endif // RAPIDJSON_STATIC_ASSERT + +// Adopt C++03 implementation from boost +#ifndef RAPIDJSON_STATIC_ASSERT +#ifndef __clang__ //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#endif RAPIDJSON_NAMESPACE_BEGIN template struct STATIC_ASSERTION_FAILURE; template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; -template struct StaticAssertTest {}; +template struct StaticAssertTest {}; RAPIDJSON_NAMESPACE_END -#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) -#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) -#define RAPIDJSON_DO_JOIN2(X, Y) X##Y - #if defined(__GNUC__) #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) #else #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif +#ifndef __clang__ //!@endcond +#endif /*! \def RAPIDJSON_STATIC_ASSERT \brief (Internal) macro to check for conditions at compile-time @@ -376,6 +451,35 @@ RAPIDJSON_NAMESPACE_END typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif // RAPIDJSON_STATIC_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY + +//! Compiler branching hint for expression with high probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression likely to be true. +*/ +#ifndef RAPIDJSON_LIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define RAPIDJSON_LIKELY(x) (x) +#endif +#endif + +//! Compiler branching hint for expression with low probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression unlikely to be true. +*/ +#ifndef RAPIDJSON_UNLIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define RAPIDJSON_UNLIKELY(x) (x) +#endif #endif /////////////////////////////////////////////////////////////////////////////// @@ -438,8 +542,12 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS #if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS __has_feature(cxx_rvalue_references) && \ +#if __has_feature(cxx_rvalue_references) && \ (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1600) @@ -470,6 +578,17 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 #endif +#ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#else +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RANGE_FOR + //!@endcond /////////////////////////////////////////////////////////////////////////////// @@ -477,7 +596,7 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_NEW ///! customization point for global \c new -#define RAPIDJSON_NEW(x) new x +#define RAPIDJSON_NEW(TypeName) new TypeName #endif #ifndef RAPIDJSON_DELETE ///! customization point for global \c delete @@ -485,10 +604,7 @@ RAPIDJSON_NAMESPACE_END #endif /////////////////////////////////////////////////////////////////////////////// -// Allocators and Encodings - -#include "allocators.h" -#include "encodings.h" +// Type /*! \namespace rapidjson \brief main RapidJSON namespace @@ -496,148 +612,6 @@ RAPIDJSON_NAMESPACE_END */ RAPIDJSON_NAMESPACE_BEGIN -/////////////////////////////////////////////////////////////////////////////// -// Stream - -/*! \class rapidjson::Stream - \brief Concept for reading and writing characters. - - For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). - - For write-only stream, only need to implement Put() and Flush(). - -\code -concept Stream { - typename Ch; //!< Character type of the stream. - - //! Read the current character from stream without moving the read cursor. - Ch Peek() const; - - //! Read the current character from stream and moving the read cursor to next character. - Ch Take(); - - //! Get the current read cursor. - //! \return Number of characters read from start. - size_t Tell(); - - //! Begin writing operation at the current read pointer. - //! \return The begin writer pointer. - Ch* PutBegin(); - - //! Write a character. - void Put(Ch c); - - //! Flush the buffer. - void Flush(); - - //! End the writing operation. - //! \param begin The begin write pointer returned by PutBegin(). - //! \return Number of characters written. - size_t PutEnd(Ch* begin); -} -\endcode -*/ - -//! Provides additional information for stream. -/*! - By using traits pattern, this type provides a default configuration for stream. - For custom stream, this type can be specialized for other configuration. - See TEST(Reader, CustomStringStream) in readertest.cpp for example. -*/ -template -struct StreamTraits { - //! Whether to make local copy of stream for optimization during parsing. - /*! - By default, for safety, streams do not use local copy optimization. - Stream that can be copied fast should specialize this, like StreamTraits. - */ - enum { copyOptimization = 0 }; -}; - -//! Put N copies of a character to a stream. -template -inline void PutN(Stream& stream, Ch c, size_t n) { - for (size_t i = 0; i < n; i++) - stream.Put(c); -} - -/////////////////////////////////////////////////////////////////////////////// -// StringStream - -//! Read-only string stream. -/*! \note implements Stream concept -*/ -template -struct GenericStringStream { - typedef typename Encoding::Ch Ch; - - GenericStringStream(const Ch *src) : src_(src), head_(src) {} - - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast(src_ - head_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! String stream with UTF8 encoding. -typedef GenericStringStream > StringStream; - -/////////////////////////////////////////////////////////////////////////////// -// InsituStringStream - -//! A read-write string stream. -/*! This string stream is particularly designed for in-situ parsing. - \note implements Stream concept -*/ -template -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; - - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return static_cast(src_ - head_); } - - // Write - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - - Ch* PutBegin() { return dst_ = src_; } - size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } - void Flush() {} - - Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } - void Pop(size_t count) { dst_ -= count; } - - Ch* src_; - Ch* dst_; - Ch* head_; -}; - -template -struct StreamTraits > { - enum { copyOptimization = 1 }; -}; - -//! Insitu string stream with UTF8 encoding. -typedef GenericInsituStringStream > InsituStringStream; - -/////////////////////////////////////////////////////////////////////////////// -// Type - //! Type of JSON value enum Type { kNullType = 0, //!< null diff --git a/contrib/rapidjson/include/rapidjson/reader.h b/contrib/rapidjson/include/rapidjson/reader.h index c5ecf4be5..120c31115 100644 --- a/contrib/rapidjson/include/rapidjson/reader.h +++ b/contrib/rapidjson/include/rapidjson/reader.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_READER_H_ @@ -17,11 +17,13 @@ /*! \file reader.h */ -#include "rapidjson.h" -#include "encodings.h" +#include "allocators.h" +#include "stream.h" +#include "encodedstream.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strtod.h" +#include #if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) #include @@ -31,6 +33,8 @@ #include #elif defined(RAPIDJSON_SSE2) #include +#elif defined(RAPIDJSON_NEON) +#include #endif #ifdef _MSC_VER @@ -39,6 +43,13 @@ RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant RAPIDJSON_DIAG_OFF(4702) // unreachable code #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(old-style-cast) +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) @@ -49,7 +60,7 @@ RAPIDJSON_DIAG_OFF(effc++) #ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ - if (HasParseError()) { return value; } \ + if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ RAPIDJSON_MULTILINEMACRO_END #endif #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ @@ -120,7 +131,7 @@ RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // ParseFlag -/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS +/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS \ingroup RAPIDJSON_CONFIG \brief User-defined kParseDefaultFlags definition. @@ -140,6 +151,10 @@ enum ParseFlag { kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. + kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. + kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; @@ -148,7 +163,7 @@ enum ParseFlag { /*! \class rapidjson::Handler \brief Concept for receiving events from GenericReader upon parsing. - The functions return true if no error occurs. If they return false, + The functions return true if no error occurs. If they return false, the event publisher should terminate the process. \code concept Handler { @@ -161,6 +176,8 @@ concept Handler { bool Int64(int64_t i); bool Uint64(uint64_t i); bool Double(double d); + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType length, bool copy); bool String(const Ch* str, SizeType length, bool copy); bool StartObject(); bool Key(const Ch* str, SizeType length, bool copy); @@ -191,6 +208,8 @@ struct BaseReaderHandler { bool Int64(int64_t) { return static_cast(*this).Default(); } bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } @@ -248,10 +267,17 @@ void SkipWhitespace(InputStream& is) { internal::StreamLocalCopy copy(is); InputStream& s(copy.s); - while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') + typename InputStream::Ch c; + while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') s.Take(); } +inline const char* SkipWhitespace(const char* p, const char* end) { + while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + return p; +} + #ifdef RAPIDJSON_SSE42 //! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. inline const char *SkipWhitespace_SIMD(const char* p) { @@ -262,7 +288,7 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; @@ -271,23 +297,37 @@ inline const char *SkipWhitespace_SIMD(const char* p) { // The rest of string using SIMD static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_load_si128((const __m128i *)&whitespace[0]); + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); for (;; p += 16) { - const __m128i s = _mm_load_si128((const __m128i *)p); - const unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; } } +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The middle of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; + } + + return SkipWhitespace(p, end); +} + #elif defined(RAPIDJSON_SSE2) //! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. @@ -299,7 +339,7 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & ~15); + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; @@ -307,24 +347,22 @@ inline const char *SkipWhitespace_SIMD(const char* p) { return p; // The rest of string - static const char whitespaces[4][17] = { - " ", - "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", - "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r", - "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"}; + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 - const __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]); - const __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]); - const __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]); - const __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]); + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); for (;; p += 16) { - const __m128i s = _mm_load_si128((const __m128i *)p); + const __m128i s = _mm_load_si128(reinterpret_cast(p)); __m128i x = _mm_cmpeq_epi8(s, w0); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = (unsigned short)~_mm_movemask_epi8(x); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); if (r != 0) { // some of characters may be non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; @@ -337,11 +375,134 @@ inline const char *SkipWhitespace_SIMD(const char* p) { } } -#endif // RAPIDJSON_SSE2 +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#elif defined(RAPIDJSON_NEON) + +//! Skip whitespace with ARM Neon instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz =__builtin_clzll(high);; + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low);; + return p + (lz >> 3); + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (; p <= end - 16; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low); + return p + (lz >> 3); + } + } + + return SkipWhitespace(p, end); +} + +#endif // RAPIDJSON_NEON #ifdef RAPIDJSON_SIMD //! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& is) { +template<> inline void SkipWhitespace(InsituStringStream& is) { is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); } @@ -349,23 +510,27 @@ template<> inline void SkipWhitespace(InsituStringStream& is) { template<> inline void SkipWhitespace(StringStream& is) { is.src_ = SkipWhitespace_SIMD(is.src_); } + +template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { + is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); +} #endif // RAPIDJSON_SIMD /////////////////////////////////////////////////////////////////////////////// // GenericReader //! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. -/*! GenericReader parses JSON text from a stream, and send events synchronously to an +/*! GenericReader parses JSON text from a stream, and send events synchronously to an object implementing Handler concept. - It needs to allocate a stack for storing a single decoded string during + It needs to allocate a stack for storing a single decoded string during non-destructive parsing. - For in-situ parsing, the decoded string is directly written to the source + For in-situ parsing, the decoded string is directly written to the source text string, no temporary buffer is required. A GenericReader object can be reused for parsing multiple JSON text. - + \tparam SourceEncoding Encoding of the input stream. \tparam TargetEncoding Encoding of the parse output. \tparam StackAllocator Allocator type for stack. @@ -398,9 +563,10 @@ public: ClearStackOnExit scope(*this); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - if (is.Peek() == '\0') { + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } @@ -409,9 +575,10 @@ public: RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - if (is.Peek() != '\0') { + if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } @@ -433,9 +600,86 @@ public: return Parse(is, handler); } + //! Initialize JSON text token-by-token parsing + /*! + */ + void IterativeParseInit() { + parseResult_.Clear(); + state_ = IterativeParsingStartState; + } + + //! Parse one token from JSON text + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + bool IterativeParseNext(InputStream& is, Handler& handler) { + while (RAPIDJSON_LIKELY(is.Peek() != '\0')) { + SkipWhitespaceAndComments(is); + + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state_, t); + IterativeParsingState d = Transit(state_, t, n, is, handler); + + // If we've finished or hit an error... + if (RAPIDJSON_UNLIKELY(IsIterativeParsingCompleteState(d))) { + // Report errors. + if (d == IterativeParsingErrorState) { + HandleError(state_, is); + return false; + } + + // Transition to the finish state. + RAPIDJSON_ASSERT(d == IterativeParsingFinishState); + state_ = d; + + // If StopWhenDone is not set... + if (!(parseFlags & kParseStopWhenDoneFlag)) { + // ... and extra non-whitespace data is found... + SkipWhitespaceAndComments(is); + if (is.Peek() != '\0') { + // ... this is considered an error. + HandleError(state_, is); + return false; + } + } + + // Success! We are done! + return true; + } + + // Transition to the new state. + state_ = d; + + // If we parsed anything other than a delimiter, we invoked the handler, so we can return true now. + if (!IsIterativeParsingDelimiterState(n)) + return true; + } + + // We reached the end of file. + stack_.Clear(); + + if (state_ != IterativeParsingFinishState) { + HandleError(state_, is); + return false; + } + + return true; + } + + //! Check if token-by-token parsing JSON text is complete + /*! \return Whether the JSON has been fully decoded. + */ + RAPIDJSON_FORCEINLINE bool IterativeParseComplete() { + return IsIterativeParsingCompleteState(state_); + } + //! Whether a parse error has occured in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } - + //! Get the \ref ParseErrorCode of last parsing. ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } @@ -462,52 +706,98 @@ private: ClearStackOnExit& operator=(const ClearStackOnExit&); }; + template + void SkipWhitespaceAndComments(InputStream& is) { + SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) { + while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { + if (Consume(is, '*')) { + while (true) { + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + else if (Consume(is, '*')) { + if (Consume(is, '/')) + break; + } + else + is.Take(); + } + } + else if (RAPIDJSON_LIKELY(Consume(is, '/'))) + while (is.Peek() != '\0' && is.Take() != '\n') {} + else + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + SkipWhitespace(is); + } + } + } + // Parse object: { string : value, ... } template void ParseObject(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '{'); is.Take(); // Skip '{' - - if (!handler.StartObject()) + + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (is.Peek() == '}') { - is.Take(); - if (!handler.EndObject(0)) // empty object + if (Consume(is, '}')) { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } for (SizeType memberCount = 0;;) { - if (is.Peek() != '"') + if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); ParseString(is, handler, true); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (is.Take() != ':') + if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++memberCount; - switch (is.Take()) { - case ',': SkipWhitespace(is); break; - case '}': - if (!handler.EndObject(memberCount)) + switch (is.Peek()) { + case ',': + is.Take(); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; + case '}': + is.Take(); + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); + default: + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy + } + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == '}') { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } } } } @@ -517,15 +807,15 @@ private: void ParseArray(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '['); is.Take(); // Skip '[' - - if (!handler.StartArray()) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - - SkipWhitespace(is); - if (is.Peek() == ']') { - is.Take(); - if (!handler.EndArray(0)) // empty array + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } @@ -535,15 +825,28 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++elementCount; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - switch (is.Take()) { - case ',': SkipWhitespace(is); break; - case ']': - if (!handler.EndArray(elementCount)) + if (Consume(is, ',')) { + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + } + else if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == ']') { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + } } } } @@ -553,12 +856,12 @@ private: RAPIDJSON_ASSERT(is.Peek() == 'n'); is.Take(); - if (is.Take() == 'u' && is.Take() == 'l' && is.Take() == 'l') { - if (!handler.Null()) + if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { + if (RAPIDJSON_UNLIKELY(!handler.Null())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template @@ -566,12 +869,12 @@ private: RAPIDJSON_ASSERT(is.Peek() == 't'); is.Take(); - if (is.Take() == 'r' && is.Take() == 'u' && is.Take() == 'e') { - if (!handler.Bool(true)) + if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template @@ -579,20 +882,30 @@ private: RAPIDJSON_ASSERT(is.Peek() == 'f'); is.Take(); - if (is.Take() == 'a' && is.Take() == 'l' && is.Take() == 's' && is.Take() == 'e') { - if (!handler.Bool(false)) + if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { + if (RAPIDJSON_LIKELY(is.Peek() == expect)) { + is.Take(); + return true; + } + else + return false; } // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). template - unsigned ParseHex4(InputStream& is) { + unsigned ParseHex4(InputStream& is, size_t escapeOffset) { unsigned codepoint = 0; for (int i = 0; i < 4; i++) { - Ch c = is.Take(); + Ch c = is.Peek(); codepoint <<= 4; codepoint += static_cast(c); if (c >= '0' && c <= '9') @@ -602,9 +915,10 @@ private: else if (c >= 'a' && c <= 'f') codepoint -= 'a' - 10; else { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); } + is.Take(); } return codepoint; } @@ -619,7 +933,14 @@ private: *stack_.template Push() = c; ++length_; } + + RAPIDJSON_FORCEINLINE void* Push(SizeType count) { + length_ += count; + return stack_.template Push(count); + } + size_t Length() const { return length_; } + Ch* Pop() { return stack_.template Pop(length_); } @@ -638,6 +959,9 @@ private: internal::StreamLocalCopy copy(is); InputStream& s(copy.s); + RAPIDJSON_ASSERT(s.Peek() == '\"'); + s.Take(); // Skip '\"' + bool success = false; if (parseFlags & kParseInsituFlag) { typename InputStream::Ch *head = s.PutBegin(); @@ -645,7 +969,7 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; size_t length = s.PutEnd(head) - 1; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - const typename TargetEncoding::Ch* const str = (typename TargetEncoding::Ch*)head; + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); } else { @@ -656,7 +980,7 @@ private: const typename TargetEncoding::Ch* const str = stackStream.Pop(); success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); } - if (!success) + if (RAPIDJSON_UNLIKELY(!success)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); } @@ -667,74 +991,421 @@ private: //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 }; #undef Z16 //!@endcond - RAPIDJSON_ASSERT(is.Peek() == '\"'); - is.Take(); // Skip '\"' - for (;;) { + // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + Ch c = is.Peek(); - if (c == '\\') { // Escape + if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape + size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset is.Take(); - Ch e = is.Take(); - if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { - os.Put(escape[(unsigned char)e]); + Ch e = is.Peek(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { + is.Take(); + os.Put(static_cast(escape[static_cast(e)])); } - else if (e == 'u') { // Unicode - unsigned codepoint = ParseHex4(is); + else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode + is.Take(); + unsigned codepoint = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { // Handle UTF-16 surrogate pair - if (is.Take() != '\\' || is.Take() != 'u') - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); - unsigned codepoint2 = ParseHex4(is); + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, is.Tell() - 2); + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; } TEncoding::Encode(os, codepoint); } else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); } - else if (c == '"') { // Closing double quote + else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote is.Take(); os.Put('\0'); // null-terminate the string return; } - else if (c == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); - else if ((unsigned)c < 0x20) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); - else { - if (parseFlags & kParseValidateEncodingFlag ? - !Transcoder::Validate(is, os) : - !Transcoder::Transcode(is, os)) + else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); + else RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); } + else { + size_t offset = is.Tell(); + if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); + } } } - template + template + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { + // Do nothing for generic version + } + +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType length; + #ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; + #else + length = static_cast(__builtin_ffs(r) - 1); + #endif + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16, q += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + for (const char* pend = p + length; p != pend; ) + *q++ = *p++; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + p += length; + break; + } + } + + is.src_ = is.dst_ = p; + } +#elif defined(RAPIDJSON_NEON) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high);; + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low);; + length = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + vst1q_u8(reinterpret_cast(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16, q += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + length = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + for (const char* pend = p + length; p != pend; ) { + *q++ = *p++; + } + break; + } + vst1q_u8(reinterpret_cast(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + p += 8 + (lz >> 3); + break; + } + } else { + int lz = __builtin_clzll(low); + p += lz >> 3; + break; + } + } + + is.src_ = is.dst_ = p; + } +#endif // RAPIDJSON_NEON + + template class NumberStream; template - class NumberStream { + class NumberStream { public: + typedef typename InputStream::Ch Ch; + NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } - ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char) {} + size_t Tell() { return is.Tell(); } size_t Length() { return 0; } const char* Pop() { return 0; } @@ -746,17 +1417,20 @@ private: }; template - class NumberStream : public NumberStream { - typedef NumberStream Base; + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : NumberStream(reader, is), stackStream(reader.stack_) {} - ~NumberStream() {} + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put((char)Base::is.Peek()); + stackStream.Put(static_cast(Base::is.Peek())); return Base::is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char c) { + stackStream.Put(c); + } + size_t Length() { return stackStream.Length(); } const char* Pop() { @@ -768,34 +1442,48 @@ private: StackStream stackStream; }; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + + RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } + }; + template void ParseNumber(InputStream& is, Handler& handler) { internal::StreamLocalCopy copy(is); - NumberStream s(*this, copy.s); + NumberStream s(*this, copy.s); + + size_t startOffset = s.Tell(); + double d = 0.0; + bool useNanOrInf = false; // Parse minus - bool minus = false; - if (s.Peek() == '-') { - minus = true; - s.Take(); - } + bool minus = Consume(s, '-'); // Parse int: zero / ( digit1-9 *DIGIT ) unsigned i = 0; uint64_t i64 = 0; bool use64bit = false; int significandDigit = 0; - if (s.Peek() == '0') { + if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { i = 0; s.TakePush(); } - else if (s.Peek() >= '1' && s.Peek() <= '9') { + else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { i = static_cast(s.TakePush() - '0'); if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 214748364) { // 2^31 = 2147483648 - if (i != 214748364 || s.Peek() > '8') { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 + if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { i64 = i; use64bit = true; break; @@ -805,9 +1493,9 @@ private: significandDigit++; } else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i >= 429496729) { // 2^32 - 1 = 4294967295 - if (i != 429496729 || s.Peek() > '5') { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 + if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { i64 = i; use64bit = true; break; @@ -817,18 +1505,41 @@ private: significandDigit++; } } + // Parse NaN or Infinity here + else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { + if (Consume(s, 'N')) { + if (Consume(s, 'a') && Consume(s, 'N')) { + d = std::numeric_limits::quiet_NaN(); + useNanOrInf = true; + } + } + else if (RAPIDJSON_LIKELY(Consume(s, 'I'))) { + if (Consume(s, 'n') && Consume(s, 'f')) { + d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + useNanOrInf = true; + + if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) { + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } + } + + if (RAPIDJSON_UNLIKELY(!useNanOrInf)) { + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); // Parse 64bit int bool useDouble = false; - double d = 0.0; if (use64bit) { - if (minus) - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC)) // 2^63 = 9223372036854775808 - if (i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8') { - d = i64; + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { + d = static_cast(i64); useDouble = true; break; } @@ -836,10 +1547,10 @@ private: significandDigit++; } else - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999)) // 2^64 - 1 = 18446744073709551615 - if (i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5') { - d = i64; + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { + d = static_cast(i64); useDouble = true; break; } @@ -850,9 +1561,9 @@ private: // Force double for big integer if (useDouble) { - while (s.Peek() >= '0' && s.Peek() <= '9') { - if (d >= 1.7976931348623157e307) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); d = d * 10 + (s.TakePush() - '0'); } } @@ -860,11 +1571,10 @@ private: // Parse frac = decimal-point 1*DIGIT int expFrac = 0; size_t decimalPosition; - if (s.Peek() == '.') { - s.Take(); + if (Consume(s, '.')) { decimalPosition = s.Length(); - if (!(s.Peek() >= '0' && s.Peek() <= '9')) + if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); if (!useDouble) { @@ -872,8 +1582,8 @@ private: // Use i64 to store significand in 64-bit architecture if (!use64bit) i64 = i; - - while (s.Peek() >= '0' && s.Peek() <= '9') { + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path break; else { @@ -884,19 +1594,19 @@ private: } } - d = (double)i64; + d = static_cast(i64); #else // Use double to store significand in 32-bit architecture - d = use64bit ? (double)i64 : (double)i; + d = static_cast(use64bit ? i64 : i); #endif useDouble = true; } - while (s.Peek() >= '0' && s.Peek() <= '9') { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (significandDigit < 17) { d = d * 10.0 + (s.TakePush() - '0'); --expFrac; - if (d > 0.0) + if (RAPIDJSON_LIKELY(d > 0.0)) significandDigit++; } else @@ -908,38 +1618,35 @@ private: // Parse exp = e [ minus / plus ] 1*DIGIT int exp = 0; - if (s.Peek() == 'e' || s.Peek() == 'E') { + if (Consume(s, 'e') || Consume(s, 'E')) { if (!useDouble) { - d = use64bit ? i64 : i; + d = static_cast(use64bit ? i64 : i); useDouble = true; } - s.Take(); bool expMinus = false; - if (s.Peek() == '+') - s.Take(); - else if (s.Peek() == '-') { - s.Take(); + if (Consume(s, '+')) + ; + else if (Consume(s, '-')) expMinus = true; - } - if (s.Peek() >= '0' && s.Peek() <= '9') { - exp = s.Take() - '0'; + if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = static_cast(s.Take() - '0'); if (expMinus) { - while (s.Peek() >= '0' && s.Peek() <= '9') { - exp = exp * 10 + (s.Take() - '0'); + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); if (exp >= 214748364) { // Issue #313: prevent overflow exponent - while (s.Peek() >= '0' && s.Peek() <= '9') // Consume the rest of exponent + while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent s.Take(); } } } else { // positive exp int maxExp = 308 - expFrac; - while (s.Peek() >= '0' && s.Peek() <= '9') { - exp = exp * 10 + (s.Take() - '0'); - if (exp > maxExp) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, s.Tell()); + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (RAPIDJSON_UNLIKELY(exp > maxExp)) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); } } } @@ -952,34 +1659,63 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; - size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - if (useDouble) { - int p = exp + expFrac; - if (parseFlags & kParseFullPrecisionFlag) - d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); - else - d = internal::StrtodNormalPrecision(d, p); - - cont = handler.Double(minus ? -d : d); - } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(-(int64_t)i64); - else - cont = handler.Uint64(i64); + if (parseFlags & kParseNumbersAsStringsFlag) { + if (parseFlags & kParseInsituFlag) { + s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch* head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + // unable to insert the \0 character here, it will erase the comma after this number + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); } else { - if (minus) - cont = handler.Int(-(int)i); - else - cont = handler.Uint(i); + SizeType numCharsToCopy = static_cast(s.Length()); + StringStream srcStream(s.Pop()); + StackStream dstStream(stack_); + while (numCharsToCopy--) { + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + } + dstStream.Put('\0'); + const typename TargetEncoding::Ch* str = dstStream.Pop(); + const SizeType length = static_cast(dstStream.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); } } - if (!cont) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + else { + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); + } + else if (useNanOrInf) { + cont = handler.Double(d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); + } + } + } + if (RAPIDJSON_UNLIKELY(!cont)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); } // Parse any JSON value @@ -992,7 +1728,10 @@ private: case '"': ParseString(is, handler); break; case '{': ParseObject(is, handler); break; case '[': ParseArray (is, handler); break; - default : ParseNumber(is, handler); + default : + ParseNumber(is, handler); + break; + } } @@ -1000,27 +1739,29 @@ private: // States enum IterativeParsingState { - IterativeParsingStartState = 0, - IterativeParsingFinishState, - IterativeParsingErrorState, + IterativeParsingFinishState = 0, // sink states at top + IterativeParsingErrorState, // sink states at top + IterativeParsingStartState, // Object states IterativeParsingObjectInitialState, IterativeParsingMemberKeyState, - IterativeParsingKeyValueDelimiterState, IterativeParsingMemberValueState, - IterativeParsingMemberDelimiterState, IterativeParsingObjectFinishState, // Array states IterativeParsingArrayInitialState, IterativeParsingElementState, - IterativeParsingElementDelimiterState, IterativeParsingArrayFinishState, // Single value state IterativeParsingValueState, - + + // Delimiter states (at bottom) + IterativeParsingElementDelimiterState, + IterativeParsingMemberDelimiterState, + IterativeParsingKeyValueDelimiterState, + cIterativeParsingStateCount }; @@ -1064,9 +1805,9 @@ private: #undef N #undef N16 //!@endcond - - if (sizeof(Ch) == 1 || unsigned(c) < 256) - return (Token)tokenMap[(unsigned char)c]; + + if (sizeof(Ch) == 1 || static_cast(c) < 256) + return static_cast(tokenMap[static_cast(c)]); else return NumberToken; } @@ -1074,6 +1815,18 @@ private: RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { // current state x one lookahead token -> new state static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, // Start { IterativeParsingArrayInitialState, // Left bracket @@ -1088,18 +1841,6 @@ private: IterativeParsingValueState, // Null IterativeParsingValueState // Number }, - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, // ObjectInitial { IterativeParsingErrorState, // Left bracket @@ -1128,20 +1869,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, // MemberValue { IterativeParsingErrorState, // Left bracket @@ -1156,20 +1883,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, // ObjectFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1204,20 +1917,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // ElementDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, // ArrayFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1229,10 +1928,52 @@ private: IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState - } + }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, }; // End of G - return (IterativeParsingState)G[state][token]; + return static_cast(G[state][token]); } // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). @@ -1309,6 +2050,11 @@ private: case IterativeParsingObjectFinishState: { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); + return IterativeParsingErrorState; + } // Get member count. SizeType c = *stack_.template Pop(1); // If the object is not empty, count the last member. @@ -1334,6 +2080,11 @@ private: case IterativeParsingArrayFinishState: { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); + return IterativeParsingErrorState; + } // Get element count. SizeType c = *stack_.template Pop(1); // If the array is not empty, count the last element. @@ -1385,55 +2136,68 @@ private: // Error flag has been set. return; } - + switch (src) { - case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); - case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; case IterativeParsingObjectInitialState: - case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); - case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - } + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; + case IterativeParsingKeyValueDelimiterState: + case IterativeParsingArrayInitialState: + case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; + default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; + } } + RAPIDJSON_FORCEINLINE bool IsIterativeParsingDelimiterState(IterativeParsingState s) { + return s >= IterativeParsingElementDelimiterState; + } + + RAPIDJSON_FORCEINLINE bool IsIterativeParsingCompleteState(IterativeParsingState s) { + return s <= IterativeParsingErrorState; + } + template ParseResult IterativeParse(InputStream& is, Handler& handler) { parseResult_.Clear(); ClearStackOnExit scope(*this); IterativeParsingState state = IterativeParsingStartState; - - SkipWhitespace(is); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); while (is.Peek() != '\0') { Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); IterativeParsingState d = Transit(state, t, n, is, handler); - + if (d == IterativeParsingErrorState) { HandleError(state, is); break; } - + state = d; - + // Do not further consume streams if a root JSON has been parsed. if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) break; - - SkipWhitespace(is); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } - + // Handle the end of file. if (state != IterativeParsingFinishState) HandleError(state, is); - + return parseResult_; } static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. ParseResult parseResult_; + IterativeParsingState state_; }; // class GenericReader //! Reader with UTF8 encoding and default allocator. @@ -1441,6 +2205,11 @@ typedef GenericReader, UTF8<> > Reader; RAPIDJSON_NAMESPACE_END +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/contrib/rapidjson/include/rapidjson/schema.h b/contrib/rapidjson/include/rapidjson/schema.h new file mode 100644 index 000000000..abcf1a102 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/schema.h @@ -0,0 +1,2016 @@ +// Tencent is pleased to support the open source community by making RapidJSON available-> +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License-> You may obtain a copy of the License at +// +// http://opensource->org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// specific language governing permissions and limitations under the License-> + +#ifndef RAPIDJSON_SCHEMA_H_ +#define RAPIDJSON_SCHEMA_H_ + +#include "document.h" +#include "pointer.h" +#include // abs, floor + +#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 +#endif + +#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX +#include "internal/regex.h" +#elif RAPIDJSON_SCHEMA_USE_STDREGEX +#include +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX +#define RAPIDJSON_SCHEMA_HAS_REGEX 1 +#else +#define RAPIDJSON_SCHEMA_HAS_REGEX 0 +#endif + +#ifndef RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_VERBOSE 0 +#endif + +#if RAPIDJSON_SCHEMA_VERBOSE +#include "stringbuffer.h" +#endif + +RAPIDJSON_DIAG_PUSH + +#if defined(__GNUC__) +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(weak-vtables) +RAPIDJSON_DIAG_OFF(exit-time-destructors) +RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) +RAPIDJSON_DIAG_OFF(variadic-macros) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Verbose Utilities + +#if RAPIDJSON_SCHEMA_VERBOSE + +namespace internal { + +inline void PrintInvalidKeyword(const char* keyword) { + printf("Fail keyword: %s\n", keyword); +} + +inline void PrintInvalidKeyword(const wchar_t* keyword) { + wprintf(L"Fail keyword: %ls\n", keyword); +} + +inline void PrintInvalidDocument(const char* document) { + printf("Fail document: %s\n\n", document); +} + +inline void PrintInvalidDocument(const wchar_t* document) { + wprintf(L"Fail document: %ls\n\n", document); +} + +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { + printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +} + +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { + wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +} // namespace internal + +#endif // RAPIDJSON_SCHEMA_VERBOSE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_INVALID_KEYWORD_RETURN + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) +#else +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) +#endif + +#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + context.invalidKeyword = keyword.GetString();\ + RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ + return false;\ +RAPIDJSON_MULTILINEMACRO_END + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations + +template +class GenericSchemaDocument; + +namespace internal { + +template +class Schema; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaValidator + +class ISchemaValidator { +public: + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaStateFactory + +template +class ISchemaStateFactory { +public: + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Hasher + +// For comparison of compound value +template +class Hasher { +public: + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) { + Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) { + WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) { + uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) { + uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const { + RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } + +private: + static const size_t kDefaultSize = 256; + struct Number { + union U { + uint64_t u; + int64_t i; + }u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) { + // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) { + static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidationContext + +template +struct SchemaValidationContext { + typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType { + kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : + factory(f), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() { + if (hasher) + factory.DestroryHasher(hasher); + if (validators) { + for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) { + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Schema + +template +class Schema { +public: + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + typeless_(schemaDocument->GetTypeless()), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false) + { + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) { + type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) { + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; + char buffer[256 + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); + } + } + + if (schemaDocument) { + AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } + + if (const ValueType* v = GetMember(value, GetNotString())) { + schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } + + // Object + + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { + // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); + + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); + + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); + } + + if (allProperties.Size() > 0) { + propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) { + new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = typeless_; + } + } + } + + if (properties && properties->IsObject()) { + PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { + SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { + PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { + new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) { + SizeType index; + if (FindPropertyIndex(*itr, &index)) { + properties_[index].required = true; + hasRequired_ = true; + } + } + + if (dependencies && dependencies->IsObject()) { + PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) { + if (itr->value.IsArray()) { + properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { + SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; + } + } + else if (itr->value.IsObject()) { + hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } + } + } + } + + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { + if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) { + PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) { // Tuple validation + itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } + } + + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); + + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { + if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); + } + + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); + + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); + + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); + + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); + + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); + + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); + + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + } + + ~Schema() { + AllocatorType::Free(enum_); + if (properties_) { + for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); + } + if (patternProperties_) { + for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); + } + AllocatorType::Free(itemsTuple_); +#if RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) { + pattern_->~RegexType(); + AllocatorType::Free(pattern_); + } +#endif + } + + bool BeginValue(Context& context) const { + if (context.inArray) { + if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) { + if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = typeless_; + else + RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + else + context.valueSchema = typeless_; + + context.arrayElementIndex++; + } + return true; + } + + RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + if (context.patternPropertiesValidatorCount > 0) { + bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) { + patternValid = false; + break; + } + + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { + if (!patternValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { + if (!patternValid || !otherValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + + if (enum_) { + const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); + foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + + if (anyOf_.schemas) { + for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); + foundAny:; + } + + if (oneOf_.schemas) { + bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) { + if (oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + else + oneValid = true; + } + if (!oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } + + if (not_ && context.validators[notValidatorIndex_]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + + return true; + } + + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Int(Context& context, int i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint(Context& context, unsigned u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint64(Context& context, uint64_t u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Double(Context& context, double d) const { + if (!(type_ & (1 << kNumberSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; + + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; + + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; + + return CreateParallelValidator(context); + } + + bool String(Context& context, const Ch* str, SizeType length, bool) const { + if (!(type_ & (1 << kStringSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) { + SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) { + if (count < minLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + if (count > maxLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (hasDependencies_ || hasRequired_) { + context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) { // pre-allocate schema array + SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const { + if (patternProperties_) { + context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + context.valueSchema = typeless_; + } + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { + if (context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) { + if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) { + context.valueSchema = typeless_; + return true; + } + + if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + + return true; + } + + bool EndObject(Context& context, SizeType memberCount) const { + if (hasRequired_) + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required) + if (!context.propertyExist[index]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + + if (memberCount < minProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + + if (memberCount > maxProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + + if (hasDependencies_) { + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) + if (context.propertyExist[sourceIndex]) { + if (properties_[sourceIndex].dependencies) { + for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + else if (properties_[sourceIndex].dependenciesSchema) + if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + } + + return true; + } + + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + context.arrayElementIndex = 0; + context.inArray = true; + + return CreateParallelValidator(context); + } + + bool EndArray(Context& context, SizeType elementCount) const { + context.inArray = false; + + if (elementCount < minItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + + if (elementCount > maxItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + + return true; + } + + // Generate functions for string literal according to Ch +#define RAPIDJSON_STRING_(name, ...) \ + static const ValueType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ + return v;\ + } + + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + RAPIDJSON_STRING_(Not, 'n', 'o', 't') + RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + +#undef RAPIDJSON_STRING_ + +private: + enum SchemaValueType { + kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + typedef internal::GenericRegex RegexType; +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + typedef std::basic_regex RegexType; +#else + typedef char RegexType; +#endif + + struct SchemaArray { + SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + template + void AddUniqueElement(V1& a, const V2& v) { + for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) { + typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { + if (const ValueType* v = GetMember(value, name)) { + if (v->IsArray() && v->Size() > 0) { + PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); + if (!r->IsValid()) { + r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { + GenericRegexSearch rs(*pattern); + return rs.Search(str); + } +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) + try { + return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) { + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { + std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } +#else + template + RegexType* CreatePattern(const ValueType&) { return 0; } + + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } +#endif // RAPIDJSON_SCHEMA_USE_STDREGEX + + void AddType(const ValueType& type) { + if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const { + if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) { + RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) { + for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { + for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { + SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { + *outIndex = index; + return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsInt64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsUint64()) { + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsInt64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsUint64()) { } + /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckUint(Context& context, uint64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsUint64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsUint64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (i % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const { + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const { + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const { + double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + return true; + } + + struct Property { + Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty { + PatternProperty() : schema(), pattern() {} + ~PatternProperty() { + if (pattern) { + pattern->~RegexType(); + AllocatorType::Free(pattern); + } + } + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + const SchemaType* typeless_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; +}; + +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = static_cast(buffer[i]); + } +}; + +// Partial specialized version for char to prevent buffer copying. +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + if (sizeof(SizeType) == 4) { + char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else { + char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// IGenericRemoteSchemaDocumentProvider + +template +class IGenericRemoteSchemaDocumentProvider { +public: + typedef typename SchemaDocumentType::Ch Ch; + + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaDocument + +//! JSON schema document. +/*! + A JSON schema document is a compiled version of a JSON schema. + It is basically a tree of internal::Schema. + + \note This is an immutable class (i.e. its instance cannot be modified after construction). + \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. + \tparam Allocator Allocator type for allocating memory of this document. +*/ +template +class GenericSchemaDocument { +public: + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + typeless_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + + typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); + new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) { + SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) { + if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) { + new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); + } + } + else if (refEntry->schema) + *refEntry->schema = typeless_; + + refEntry->~SchemaRefEntry(); + } + + RAPIDJSON_ASSERT(root_ != 0); + + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + typeless_(rhs.typeless_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)) + { + rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.typeless_ = 0; + } +#endif + + //! Destructor + ~GenericSchemaDocument() { + while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); + + if (typeless_) { + typeless_->~SchemaType(); + Allocator::Free(typeless_); + } + + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } + +private: + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry { + SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry { + SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() { + if (owned) { + schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + if (schema) + *schema = typeless_; + + if (v.GetType() == kObjectType) { + const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) { + if (!HandleRefSchema(pointer, schema, v, document)) { + SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); + if (schema) + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { + static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) { + SizeType len = itr->value.GetStringLength(); + if (len > 0) { + const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) { // Remote reference, resolve immediately + if (remoteProvider_) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { + if (schema) + *schema = sc; + return true; + } + } + } + } + } + else if (s[i] == '#') { // Local reference, defer resolution + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; + + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + const SchemaType* GetTypeless() const { return typeless_; } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + SchemaType* typeless_; + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref +}; + +//! GenericSchemaDocument using Value type. +typedef GenericSchemaDocument SchemaDocument; +//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaValidator + +//! JSON Schema Validator. +/*! + A SAX style JSON schema validator. + It uses a \c GenericSchemaDocument to validate SAX events. + It delegates the incoming SAX events to an output handler. + The default output handler does nothing. + It can be reused multiple times by calling \c Reset(). + + \tparam SchemaDocumentType Type of schema document. + \tparam OutputHandler Type of output handler. Default handler does nothing. + \tparam StateAllocator Allocator for storing the internal validation states. +*/ +template < + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> +class GenericSchemaValidator : + public internal::ISchemaStateFactory, + public internal::ISchemaValidator +{ +public: + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(&outputHandler), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Destructor. + ~GenericSchemaValidator() { + Reset(); + RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() { + while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const { + return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const { + return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const { + return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + internal::PrintInvalidDocument(documentStack_.template Bottom());\ +RAPIDJSON_MULTILINEMACRO_END +#else +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() +#endif + +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ + if (!valid_) return false; \ + if (!BeginValue() || !CurrentSchema().method arg1) {\ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ + return valid_ = false;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ + for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ + if (context->hasher)\ + static_cast(context->hasher)->method arg2;\ + if (context->validators)\ + for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ + static_cast(context->validators[i_])->method arg2;\ + if (context->patternPropertiesValidators)\ + for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ + static_cast(context->patternPropertiesValidators[i_])->method arg2;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ + return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) + +#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ + RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) + + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } + bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = !outputHandler_ || outputHandler_->StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) { + if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = !outputHandler_ || outputHandler_->StartArray(); + } + + bool EndArray(SizeType elementCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } + +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ +#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ +#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ + + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, +#if RAPIDJSON_SCHEMA_VERBOSE + depth_ + 1, +#endif + &GetStateAllocator()); + } + + virtual void DestroySchemaValidator(ISchemaValidator* validator) { + GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } + + virtual void* CreateHasher() { + return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } + + virtual uint64_t GetHashCode(void* hasher) { + return static_cast(hasher)->GetHashCode(); + } + + virtual void DestroryHasher(void* hasher) { + HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } + + virtual void* MallocState(size_t size) { + return GetStateAllocator().Malloc(size); + } + + virtual void FreeState(void* p) { + StateAllocator::Free(p); + } + +private: + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; + + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth, +#endif + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(depth) +#endif + { + } + + StateAllocator& GetStateAllocator() { + if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator)(); + return *stateAllocator_; + } + + bool BeginValue() { + if (schemaStack_.Empty()) + PushSchema(root_); + else { + if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; + + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + RAPIDJSON_ASSERT(CurrentContext().valueSchema); + PushSchema(*CurrentContext().valueSchema); + + if (count > 0) { + CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } + + CurrentContext().arrayUniqueness = valueUniqueness; + } + return true; + } + + bool EndValue() { + if (!CurrentSchema().EndValue(CurrentContext())) + return false; + +#if RAPIDJSON_SCHEMA_VERBOSE + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); +#endif + + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) { + Context& context = CurrentContext(); + if (context.valueUniqueness) { + HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) + RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) { + documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) { + if (str[i] == '~') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } + + RAPIDJSON_FORCEINLINE void PopSchema() { + Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { + a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + OutputHandler* outputHandler_; + bool valid_; +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth_; +#endif +}; + +typedef GenericSchemaValidator SchemaValidator; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidatingReader + +//! A helper class for parsing with validation. +/*! + This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). + + \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam SourceEncoding Encoding of the input stream. + \tparam SchemaDocumentType Type of schema document. + \tparam StackAllocator Allocator type for stack. +*/ +template < + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader { +public: + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} + + template + bool operator()(Handler& handler) { + GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) { + invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + } + else { + invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + +private: + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + bool isValid_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_SCHEMA_H_ diff --git a/contrib/rapidjson/include/rapidjson/stream.h b/contrib/rapidjson/include/rapidjson/stream.h new file mode 100644 index 000000000..fef82c252 --- /dev/null +++ b/contrib/rapidjson/include/rapidjson/stream.h @@ -0,0 +1,179 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "rapidjson.h" + +#ifndef RAPIDJSON_STREAM_H_ +#define RAPIDJSON_STREAM_H_ + +#include "encodings.h" + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Reserve n characters for writing to a stream. +template +inline void PutReserve(Stream& stream, size_t count) { + (void)stream; + (void)count; +} + +//! Write character to a stream, presuming buffer is reserved. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { + stream.Put(c); +} + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + PutReserve(stream, n); + for (size_t i = 0; i < n; i++) + PutUnsafe(stream, c); +} + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_STREAM_H_ diff --git a/contrib/rapidjson/include/rapidjson/stringbuffer.h b/contrib/rapidjson/include/rapidjson/stringbuffer.h index 1c9c80b79..4e38b82c3 100644 --- a/contrib/rapidjson/include/rapidjson/stringbuffer.h +++ b/contrib/rapidjson/include/rapidjson/stringbuffer.h @@ -15,7 +15,8 @@ #ifndef RAPIDJSON_STRINGBUFFER_H_ #define RAPIDJSON_STRINGBUFFER_H_ -#include "rapidjson.h" +#include "stream.h" +#include "internal/stack.h" #if RAPIDJSON_HAS_CXX11_RVALUE_REFS #include // std::move @@ -23,6 +24,11 @@ #include "internal/stack.h" +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Represents an in-memory output stream. @@ -48,6 +54,7 @@ public: #endif void Put(Ch c) { *stack_.template Push() = c; } + void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } void Flush() {} void Clear() { stack_.Clear(); } @@ -57,7 +64,10 @@ public: stack_.ShrinkToFit(); stack_.template Pop(1); } + + void Reserve(size_t count) { stack_.template Reserve(count); } Ch* Push(size_t count) { return stack_.template Push(count); } + Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } void Pop(size_t count) { stack_.template Pop(count); } const Ch* GetString() const { @@ -68,8 +78,12 @@ public: return stack_.template Bottom(); } + //! Get the size of string in bytes in the string buffer. size_t GetSize() const { return stack_.GetSize(); } + //! Get the length of string in Ch in the string buffer. + size_t GetLength() const { return stack_.GetSize() / sizeof(Ch); } + static const size_t kDefaultCapacity = 256; mutable internal::Stack stack_; @@ -82,6 +96,16 @@ private: //! String buffer with UTF8 encoding typedef GenericStringBuffer > StringBuffer; +template +inline void PutReserve(GenericStringBuffer& stream, size_t count) { + stream.Reserve(count); +} + +template +inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { + stream.PutUnsafe(c); +} + //! Implement specialized version of PutN() with memset() for better performance. template<> inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { @@ -90,4 +114,8 @@ inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/contrib/rapidjson/include/rapidjson/writer.h b/contrib/rapidjson/include/rapidjson/writer.h index e1eea38b9..e610ebb60 100644 --- a/contrib/rapidjson/include/rapidjson/writer.h +++ b/contrib/rapidjson/include/rapidjson/writer.h @@ -15,7 +15,8 @@ #ifndef RAPIDJSON_WRITER_H_ #define RAPIDJSON_WRITER_H_ -#include "rapidjson.h" +#include "stream.h" +#include "internal/meta.h" #include "internal/stack.h" #include "internal/strfunc.h" #include "internal/dtoa.h" @@ -23,8 +24,16 @@ #include "stringbuffer.h" #include // placement new -#if RAPIDJSON_HAS_STDSTRING -#include +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#elif defined(RAPIDJSON_NEON) +#include #endif #ifdef _MSC_VER @@ -32,8 +41,36 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant #endif +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN +/////////////////////////////////////////////////////////////////////////////// +// WriteFlag + +/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kWriteDefaultFlags definition. + + User can define this as any \c WriteFlag combinations. +*/ +#ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS +#define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags +#endif + +//! Combination of writeFlags +enum WriteFlag { + kWriteNoFlags = 0, //!< No flags are set. + kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. + kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. + kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS +}; + //! JSON writer /*! Writer implements the concept Handler. It generates JSON text by events to an output os. @@ -50,11 +87,13 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam StackAllocator Type of allocator for allocating memory of stack. \note implements Handler concept */ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator> +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> class Writer { public: typedef typename SourceEncoding::Ch Ch; + static const int kDefaultMaxDecimalPlaces = 324; + //! Constructor /*! \param os Output stream. \param stackAllocator User supplied allocator. If it is null, it will create a private one. @@ -62,11 +101,18 @@ public: */ explicit Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), hasRoot_(false) {} + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} explicit Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), hasRoot_(false) {} + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Writer(Writer&& rhs) : + os_(rhs.os_), level_stack_(std::move(rhs.level_stack_)), maxDecimalPlaces_(rhs.maxDecimalPlaces_), hasRoot_(rhs.hasRoot_) { + rhs.os_ = 0; + } +#endif //! Reset the writer with a new stream. /*! @@ -100,29 +146,66 @@ public: return hasRoot_ && level_stack_.Empty(); } + int GetMaxDecimalPlaces() const { + return maxDecimalPlaces_; + } + + //! Sets the maximum number of decimal places for double output. + /*! + This setting truncates the output with specified number of decimal places. + + For example, + + \code + writer.SetMaxDecimalPlaces(3); + writer.StartArray(); + writer.Double(0.12345); // "0.123" + writer.Double(0.0001); // "0.0" + writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) + writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) + writer.EndArray(); + \endcode + + The default setting does not truncate any decimal places. You can restore to this setting by calling + \code + writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); + \endcode + */ + void SetMaxDecimalPlaces(int maxDecimalPlaces) { + maxDecimalPlaces_ = maxDecimalPlaces; + } + /*!@name Implementation of Handler \see Handler */ //@{ - bool Null() { Prefix(kNullType); return WriteNull(); } - bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } - bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } - bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } - bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } - bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } + bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } + bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } + bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } + bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } //! Writes the given \c double value to the stream /*! \param d The value to be written. \return Whether it is succeed. */ - bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } + bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); + (void)copy; + Prefix(kNumberType); + return EndValue(WriteString(str, length)); + } bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; Prefix(kStringType); - return WriteString(str, length); + return EndValue(WriteString(str, length)); } #if RAPIDJSON_HAS_STDSTRING @@ -138,16 +221,21 @@ public: } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) + { + return Key(str.data(), SizeType(str.size())); + } +#endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value level_stack_.template Pop(1); - bool ret = WriteEndObject(); - if (level_stack_.Empty()) // end of json text - os_->Flush(); - return ret; + return EndValue(WriteEndObject()); } bool StartArray() { @@ -161,10 +249,7 @@ public: RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); level_stack_.template Pop(1); - bool ret = WriteEndArray(); - if (level_stack_.Empty()) // end of json text - os_->Flush(); - return ret; + return EndValue(WriteEndArray()); } //@} @@ -172,11 +257,33 @@ public: //@{ //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - + bool String(const Ch* const& str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* const& str) { return Key(str, internal::StrLen(str)); } + //@} + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + Prefix(type); + return EndValue(WriteRawValue(json, length)); + } + + //! Flush the output stream. + /*! + Allows the user to flush the output stream immediately. + */ + void Flush() { + os_->Flush(); + } + protected: //! Information for each nested level struct Level { @@ -188,15 +295,18 @@ protected: static const size_t kDefaultLevelDepth = 32; bool WriteNull() { - os_->Put('n'); os_->Put('u'); os_->Put('l'); os_->Put('l'); return true; + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; } bool WriteBool(bool b) { if (b) { - os_->Put('t'); os_->Put('r'); os_->Put('u'); os_->Put('e'); + PutReserve(*os_, 4); + PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); } else { - os_->Put('f'); os_->Put('a'); os_->Put('l'); os_->Put('s'); os_->Put('e'); + PutReserve(*os_, 5); + PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); } return true; } @@ -204,45 +314,69 @@ protected: bool WriteInt(int i) { char buffer[11]; const char* end = internal::i32toa(i, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteUint(unsigned u) { char buffer[10]; const char* end = internal::u32toa(u, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteInt64(int64_t i64) { char buffer[21]; const char* end = internal::i64toa(i64, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteUint64(uint64_t u64) { char buffer[20]; char* end = internal::u64toa(u64, buffer); + PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + if (!(writeFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + char buffer[25]; - char* end = internal::dtoa(d, buffer); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - os_->Put(*p); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteString(const Ch* str, SizeType length) { - static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const typename OutputStream::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; static const char escape[256] = { #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 //0 1 2 3 4 5 6 7 8 9 A B C D E F @@ -255,22 +389,27 @@ protected: #undef Z16 }; - os_->Put('\"'); + if (TargetEncoding::supportUnicode) + PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + else + PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." + + PutUnsafe(*os_, '\"'); GenericStringStream is(str); - while (is.Tell() < length) { + while (ScanWriteUnescapedString(is, length)) { const Ch c = is.Peek(); - if (!TargetEncoding::supportUnicode && (unsigned)c >= 0x80) { + if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { // Unicode escaping unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) + if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) return false; - os_->Put('\\'); - os_->Put('u'); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { - os_->Put(hexDigits[(codepoint >> 12) & 15]); - os_->Put(hexDigits[(codepoint >> 8) & 15]); - os_->Put(hexDigits[(codepoint >> 4) & 15]); - os_->Put(hexDigits[(codepoint ) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); } else { RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); @@ -278,45 +417,59 @@ protected: unsigned s = codepoint - 0x010000; unsigned lead = (s >> 10) + 0xD800; unsigned trail = (s & 0x3FF) + 0xDC00; - os_->Put(hexDigits[(lead >> 12) & 15]); - os_->Put(hexDigits[(lead >> 8) & 15]); - os_->Put(hexDigits[(lead >> 4) & 15]); - os_->Put(hexDigits[(lead ) & 15]); - os_->Put('\\'); - os_->Put('u'); - os_->Put(hexDigits[(trail >> 12) & 15]); - os_->Put(hexDigits[(trail >> 8) & 15]); - os_->Put(hexDigits[(trail >> 4) & 15]); - os_->Put(hexDigits[(trail ) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(lead ) & 15]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(trail ) & 15]); } } - else if ((sizeof(Ch) == 1 || (unsigned)c < 256) && escape[(unsigned char)c]) { + else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { is.Take(); - os_->Put('\\'); - os_->Put(escape[(unsigned char)c]); - if (escape[(unsigned char)c] == 'u') { - os_->Put('0'); - os_->Put('0'); - os_->Put(hexDigits[(unsigned char)c >> 4]); - os_->Put(hexDigits[(unsigned char)c & 0xF]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + if (escape[static_cast(c)] == 'u') { + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); + PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); } } - else - if (!Transcoder::Transcode(is, *os_)) - return false; + else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; } - os_->Put('\"'); + PutUnsafe(*os_, '\"'); return true; } + bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { + return RAPIDJSON_LIKELY(is.Tell() < length); + } + bool WriteStartObject() { os_->Put('{'); return true; } bool WriteEndObject() { os_->Put('}'); return true; } bool WriteStartArray() { os_->Put('['); return true; } bool WriteEndArray() { os_->Put(']'); return true; } + bool WriteRawValue(const Ch* json, size_t length) { + PutReserve(*os_, length); + for (size_t i = 0; i < length; i++) { + RAPIDJSON_ASSERT(json[i] != '\0'); + PutUnsafe(*os_, json[i]); + } + return true; + } + void Prefix(Type type) { (void)type; - if (level_stack_.GetSize() != 0) { // this value is not at root + if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root Level* level = level_stack_.template Top(); if (level->valueCount > 0) { if (level->inArray) @@ -334,8 +487,16 @@ protected: } } + // Flush the value if it is the top level one. + bool EndValue(bool ret) { + if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + Flush(); + return ret; + } + OutputStream* os_; internal::Stack level_stack_; + int maxDecimalPlaces_; bool hasRoot_; private: @@ -350,7 +511,7 @@ template<> inline bool Writer::WriteInt(int i) { char *buffer = os_->Push(11); const char* end = internal::i32toa(i, buffer); - os_->Pop(11 - (end - buffer)); + os_->Pop(static_cast(11 - (end - buffer))); return true; } @@ -358,7 +519,7 @@ template<> inline bool Writer::WriteUint(unsigned u) { char *buffer = os_->Push(10); const char* end = internal::u32toa(u, buffer); - os_->Pop(10 - (end - buffer)); + os_->Pop(static_cast(10 - (end - buffer))); return true; } @@ -366,7 +527,7 @@ template<> inline bool Writer::WriteInt64(int64_t i64) { char *buffer = os_->Push(21); const char* end = internal::i64toa(i64, buffer); - os_->Pop(21 - (end - buffer)); + os_->Pop(static_cast(21 - (end - buffer))); return true; } @@ -374,22 +535,177 @@ template<> inline bool Writer::WriteUint64(uint64_t u) { char *buffer = os_->Push(20); const char* end = internal::u64toa(u, buffer); - os_->Pop(20 - (end - buffer)); + os_->Pop(static_cast(20 - (end - buffer))); return true; } template<> inline bool Writer::WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). + if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + char *buffer = os_->Push(25); - char* end = internal::dtoa(d, buffer); - os_->Pop(25 - (end - buffer)); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + os_->Pop(static_cast(25 - (end - buffer))); return true; } +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (; p != endAligned; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType len; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + len = offset; +#else + len = static_cast(__builtin_ffs(r) - 1); +#endif + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#elif defined(RAPIDJSON_NEON) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (; p != endAligned; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType len = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + len = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + len = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + vst1q_u8(reinterpret_cast(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#endif // RAPIDJSON_NEON + RAPIDJSON_NAMESPACE_END #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + #endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/contrib/rapidjson/license.txt b/contrib/rapidjson/license.txt index 879293afa..7ccc161c8 100644 --- a/contrib/rapidjson/license.txt +++ b/contrib/rapidjson/license.txt @@ -3,7 +3,7 @@ Tencent is pleased to support the open source community by making RapidJSON avai Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. +If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. A copy of the MIT License is included in this file. Other dependencies and licenses: diff --git a/contrib/rapidjson/readme.md b/contrib/rapidjson/readme.md index 19da38667..b833a98e8 100644 --- a/contrib/rapidjson/readme.md +++ b/contrib/rapidjson/readme.md @@ -1,17 +1,17 @@ -![](doc/logo/rapidjson.png) +![RapidJSON logo](doc/logo/rapidjson.png) -![](https://img.shields.io/badge/release-v1.0.2-blue.png) +![Release version](https://img.shields.io/badge/release-v1.1.0-blue.svg) -## A fast JSON parser/generator for C++ with both SAX/DOM style API +## A fast JSON parser/generator for C++ with both SAX/DOM style API Tencent is pleased to support the open source community by making RapidJSON available. Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -* [RapidJSON GitHub](https://github.com/miloyip/rapidjson/) +* [RapidJSON GitHub](https://github.com/Tencent/rapidjson/) * RapidJSON Documentation - * [English](http://miloyip.github.io/rapidjson/) - * [简体中文](http://miloyip.github.io/rapidjson/zh-cn/) + * [English](http://rapidjson.org/) + * [简体中文](http://rapidjson.org/zh-cn/) * [GitBook](https://www.gitbook.com/book/miloyip/rapidjson/) with downloadable PDF/EPUB/MOBI, without API reference. ## Build status @@ -20,33 +20,43 @@ Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights | :---------------: | :-----------------: | :-------------------: | | ![lin-badge] | ![win-badge] | ![cov-badge] | -[lin-badge]: https://travis-ci.org/miloyip/rapidjson.png?branch=master "Travis build status" -[lin-link]: https://travis-ci.org/miloyip/rapidjson "Travis build status" -[win-badge]: https://ci.appveyor.com/api/projects/status/u658dcuwxo14a8m9/branch/master "AppVeyor build status" -[win-link]: https://ci.appveyor.com/project/miloyip/rapidjson/branch/master "AppVeyor build status" -[cov-badge]: https://coveralls.io/repos/miloyip/rapidjson/badge.png?branch=master -[cov-link]: https://coveralls.io/r/miloyip/rapidjson?branch=master +[lin-badge]: https://travis-ci.org/Tencent/rapidjson.svg?branch=master "Travis build status" +[lin-link]: https://travis-ci.org/Tencent/rapidjson "Travis build status" +[win-badge]: https://ci.appveyor.com/api/projects/status/l6qulgqahcayidrf/branch/master?svg=true "AppVeyor build status" +[win-link]: https://ci.appveyor.com/project/miloyip/rapidjson-0fdqj/branch/master "AppVeyor build status" +[cov-badge]: https://coveralls.io/repos/Tencent/rapidjson/badge.svg?branch=master "Coveralls coverage" +[cov-link]: https://coveralls.io/r/Tencent/rapidjson?branch=master "Coveralls coverage" ## Introduction RapidJSON is a JSON parser and generator for C++. It was inspired by [RapidXml](http://rapidxml.sourceforge.net/). -* RapidJSON is small but complete. It supports both SAX and DOM style API. The SAX parser is only a half thousand lines of code. +* RapidJSON is **small** but **complete**. It supports both SAX and DOM style API. The SAX parser is only a half thousand lines of code. -* RapidJSON is fast. Its performance can be comparable to `strlen()`. It also optionally supports SSE2/SSE4.2 for acceleration. +* RapidJSON is **fast**. Its performance can be comparable to `strlen()`. It also optionally supports SSE2/SSE4.2 for acceleration. -* RapidJSON is self-contained. It does not depend on external libraries such as BOOST. It even does not depend on STL. +* RapidJSON is **self-contained** and **header-only**. It does not depend on external libraries such as BOOST. It even does not depend on STL. -* RapidJSON is memory friendly. Each JSON value occupies exactly 16/20 bytes for most 32/64-bit machines (excluding text string). By default it uses a fast memory allocator, and the parser allocates memory compactly during parsing. +* RapidJSON is **memory-friendly**. Each JSON value occupies exactly 16 bytes for most 32/64-bit machines (excluding text string). By default it uses a fast memory allocator, and the parser allocates memory compactly during parsing. -* RapidJSON is Unicode friendly. It supports UTF-8, UTF-16, UTF-32 (LE & BE), and their detection, validation and transcoding internally. For example, you can read a UTF-8 file and let RapidJSON transcode the JSON strings into UTF-16 in the DOM. It also supports surrogates and "\u0000" (null character). +* RapidJSON is **Unicode-friendly**. It supports UTF-8, UTF-16, UTF-32 (LE & BE), and their detection, validation and transcoding internally. For example, you can read a UTF-8 file and let RapidJSON transcode the JSON strings into UTF-16 in the DOM. It also supports surrogates and "\u0000" (null character). More features can be read [here](doc/features.md). -JSON(JavaScript Object Notation) is a light-weight data exchange format. RapidJSON should be in fully compliance with RFC7159/ECMA-404. More information about JSON can be obtained at +JSON(JavaScript Object Notation) is a light-weight data exchange format. RapidJSON should be in fully compliance with RFC7159/ECMA-404, with optional support of relaxed syntax. More information about JSON can be obtained at * [Introducing JSON](http://json.org/) -* [RFC7159: The JavaScript Object Notation (JSON) Data Interchange Format](http://www.ietf.org/rfc/rfc7159.txt) -* [Standard ECMA-404: The JSON Data Interchange Format](http://www.ecma-international.org/publications/standards/Ecma-404.htm) +* [RFC7159: The JavaScript Object Notation (JSON) Data Interchange Format](https://tools.ietf.org/html/rfc7159) +* [Standard ECMA-404: The JSON Data Interchange Format](https://www.ecma-international.org/publications/standards/Ecma-404.htm) + +## Highlights in v1.1 (2016-8-25) + +* Added [JSON Pointer](doc/pointer.md) +* Added [JSON Schema](doc/schema.md) +* Added [relaxed JSON syntax](doc/dom.md) (comment, trailing comma, NaN/Infinity) +* Iterating array/object with [C++11 Range-based for loop](doc/tutorial.md) +* Reduce memory overhead of each `Value` from 24 bytes to 16 bytes in x86-64 architecture. + +For other changes please refer to [change log](CHANGELOG.md). ## Compatibility @@ -63,9 +73,9 @@ Users can build and run the unit tests on their platform/compiler. RapidJSON is a header-only C++ library. Just copy the `include/rapidjson` folder to system or project's include path. RapidJSON uses following software as its dependencies: -* [CMake](http://www.cmake.org) as a general build tool -* (optional)[Doxygen](http://www.doxygen.org) to build documentation -* (optional)[googletest](https://code.google.com/p/googletest/) for unit and performance testing +* [CMake](https://cmake.org/) as a general build tool +* (optional) [Doxygen](http://www.doxygen.org) to build documentation +* (optional) [googletest](https://github.com/google/googletest) for unit and performance testing To generate user documentation and run tests please proceed with the steps below: @@ -74,7 +84,7 @@ To generate user documentation and run tests please proceed with the steps below 3. Change to `build` directory and run `cmake ..` command to configure your build. Windows users can do the same with cmake-gui application. 4. On Windows, build the solution found in the build directory. On Linux, run `make` from the build directory. -On successfull build you will find compiled test and example binaries in `bin` +On successful build you will find compiled test and example binaries in `bin` directory. The generated documentation will be available in `doc/html` directory of the build tree. To run tests after finished build please run `make test` or `ctest` from your build tree. You can get detailed output using `ctest @@ -126,4 +136,25 @@ The following diagram shows the process. ![simpledom](doc/diagram/simpledom.png) -More [examples](https://github.com/miloyip/rapidjson/tree/master/example) are available. +More [examples](https://github.com/Tencent/rapidjson/tree/master/example) are available: + +* DOM API + * [tutorial](https://github.com/Tencent/rapidjson/blob/master/example/tutorial/tutorial.cpp): Basic usage of DOM API. + +* SAX API + * [simplereader](https://github.com/Tencent/rapidjson/blob/master/example/simplereader/simplereader.cpp): Dumps all SAX events while parsing a JSON by `Reader`. + * [condense](https://github.com/Tencent/rapidjson/blob/master/example/condense/condense.cpp): A command line tool to rewrite a JSON, with all whitespaces removed. + * [pretty](https://github.com/Tencent/rapidjson/blob/master/example/pretty/pretty.cpp): A command line tool to rewrite a JSON with indents and newlines by `PrettyWriter`. + * [capitalize](https://github.com/Tencent/rapidjson/blob/master/example/capitalize/capitalize.cpp): A command line tool to capitalize strings in JSON. + * [messagereader](https://github.com/Tencent/rapidjson/blob/master/example/messagereader/messagereader.cpp): Parse a JSON message with SAX API. + * [serialize](https://github.com/Tencent/rapidjson/blob/master/example/serialize/serialize.cpp): Serialize a C++ object into JSON with SAX API. + * [jsonx](https://github.com/Tencent/rapidjson/blob/master/example/jsonx/jsonx.cpp): Implements a `JsonxWriter` which stringify SAX events into [JSONx](https://www-01.ibm.com/support/knowledgecenter/SS9H2Y_7.1.0/com.ibm.dp.doc/json_jsonx.html) (a kind of XML) format. The example is a command line tool which converts input JSON into JSONx format. + +* Schema + * [schemavalidator](https://github.com/Tencent/rapidjson/blob/master/example/schemavalidator/schemavalidator.cpp) : A command line tool to validate a JSON with a JSON schema. + +* Advanced + * [prettyauto](https://github.com/Tencent/rapidjson/blob/master/example/prettyauto/prettyauto.cpp): A modified version of [pretty](https://github.com/Tencent/rapidjson/blob/master/example/pretty/pretty.cpp) to automatically handle JSON with any UTF encodings. + * [parsebyparts](https://github.com/Tencent/rapidjson/blob/master/example/parsebyparts/parsebyparts.cpp): Implements an `AsyncDocumentParser` which can parse JSON in parts, using C++11 thread. + * [filterkey](https://github.com/Tencent/rapidjson/blob/master/example/filterkey/filterkey.cpp): A command line tool to remove all values with user-specified key. + * [filterkeydom](https://github.com/Tencent/rapidjson/blob/master/example/filterkeydom/filterkeydom.cpp): Same tool as above, but it demonstrates how to use a generator to populate a `Document`. From 9eef4c16a828f35900f9f97a46e85296515c631a Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:33:27 +0300 Subject: [PATCH 244/305] utMetadata: Fix memory leak --- test/unit/utMetadata.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/utMetadata.cpp b/test/unit/utMetadata.cpp index b44de1eb2..4109b068c 100644 --- a/test/unit/utMetadata.cpp +++ b/test/unit/utMetadata.cpp @@ -79,6 +79,7 @@ TEST_F( utMetadata, allocTest ) { EXPECT_EQ( 1U, data->mNumProperties ); EXPECT_NE( nullptr, data->mKeys ); EXPECT_NE( nullptr, data->mValues ); + aiMetadata::Dealloc( data ); } TEST_F( utMetadata, get_set_pod_Test ) { From 1bee5b002564ef9b7b3f765a1247ae8991e60123 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:55:58 +0300 Subject: [PATCH 245/305] utRemoveVCProcess: Fix memory leak --- test/unit/utRemoveVCProcess.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/utRemoveVCProcess.cpp b/test/unit/utRemoveVCProcess.cpp index c78e80d3f..6caa72c11 100644 --- a/test/unit/utRemoveVCProcess.cpp +++ b/test/unit/utRemoveVCProcess.cpp @@ -70,7 +70,7 @@ TEST_F( utRevmoveVCProcess, issue1266_ProcessMeshTest_NoCrash ) { mesh->mNumVertices = 1; mesh->mColors[ 0 ] = new aiColor4D[ 2 ]; scene->mMeshes[ 0 ] = mesh; - RemoveVCProcess *process = new RemoveVCProcess; + std::unique_ptr process(new RemoveVCProcess); process->Execute( scene ); delete scene; } From 1f16ed9fd0956e73f8ff59a90204b476a56e8a2a Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 11:56:13 +0300 Subject: [PATCH 246/305] UnrealLoader: Fix IOStream leak --- code/UnrealLoader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/UnrealLoader.cpp b/code/UnrealLoader.cpp index c8382cb01..a79a2a5c5 100644 --- a/code/UnrealLoader.cpp +++ b/code/UnrealLoader.cpp @@ -157,7 +157,7 @@ void UnrealImporter::InternReadFile( const std::string& pFile, DefaultLogger::get()->debug("UNREAL: uc file is " + uc_path); // and open the files ... we can't live without them - IOStream* p = pIOHandler->Open(d_path); + std::unique_ptr p(pIOHandler->Open(d_path)); if (!p) throw DeadlyImportError("UNREAL: Unable to open _d file"); StreamReaderLE d_reader(pIOHandler->Open(d_path)); @@ -203,7 +203,7 @@ void UnrealImporter::InternReadFile( const std::string& pFile, d_reader.IncPtr(1); } - p = pIOHandler->Open(a_path); + p.reset(pIOHandler->Open(a_path)); if (!p) throw DeadlyImportError("UNREAL: Unable to open _a file"); StreamReaderLE a_reader(pIOHandler->Open(a_path)); From 61278aa408e77f81a3c2d0f2f894dd5b1c667d9f Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:09:26 +0300 Subject: [PATCH 247/305] AMFImporter: Fix memory leak --- code/AMFImporter_Postprocess.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/code/AMFImporter_Postprocess.cpp b/code/AMFImporter_Postprocess.cpp index 085336c51..789a11eb8 100644 --- a/code/AMFImporter_Postprocess.cpp +++ b/code/AMFImporter_Postprocess.cpp @@ -686,7 +686,6 @@ std::list mesh_idx; tmesh->mNumVertices = static_cast(vert_arr.size()); tmesh->mVertices = new aiVector3D[tmesh->mNumVertices]; tmesh->mColors[0] = new aiColor4D[tmesh->mNumVertices]; - tmesh->mFaces = new aiFace[face_list_cur.size()]; memcpy(tmesh->mVertices, vert_arr.data(), tmesh->mNumVertices * sizeof(aiVector3D)); memcpy(tmesh->mColors[0], col_arr.data(), tmesh->mNumVertices * sizeof(aiColor4D)); From 22b55d01a20d47def5cb7c5b204271a845bc3f7a Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:35:17 +0300 Subject: [PATCH 248/305] OpenGEXImporter: Store ChildInfo in unique_ptr so they get automatically cleaned up --- code/OpenGEXImporter.cpp | 8 ++++---- code/OpenGEXImporter.h | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index d9c58e91c..0d2ed0520 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -1231,9 +1231,9 @@ void OpenGEXImporter::pushNode( aiNode *node, aiScene *pScene ) { if( m_nodeChildMap.end() == it ) { info = new ChildInfo; m_root = info; - m_nodeChildMap[ node->mParent ] = info; + m_nodeChildMap[ node->mParent ] = std::unique_ptr(info); } else { - info = it->second; + info = it->second.get(); } info->m_children.push_back( node ); } else { @@ -1243,9 +1243,9 @@ void OpenGEXImporter::pushNode( aiNode *node, aiScene *pScene ) { NodeChildMap::iterator it( m_nodeChildMap.find( node->mParent ) ); if( m_nodeChildMap.end() == it ) { info = new ChildInfo; - m_nodeChildMap[ node->mParent ] = info; + m_nodeChildMap[ node->mParent ] = std::unique_ptr(info); } else { - info = it->second; + info = it->second.get(); } info->m_children.push_back( node ); } diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index bb6b45140..948712869 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include namespace ODDLParser { class DDLNode; @@ -179,7 +180,7 @@ private: std::list m_children; }; ChildInfo *m_root; - typedef std::map NodeChildMap; + typedef std::map > NodeChildMap; NodeChildMap m_nodeChildMap; std::vector m_meshCache; From 775f984d99a4bce513f5ce712d77f38116faef1e Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:38:44 +0300 Subject: [PATCH 249/305] OpenGEXImporter: Fix IOStream leak --- code/OpenGEXImporter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 0d2ed0520..24dabd1d4 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -307,6 +307,7 @@ void OpenGEXImporter::InternReadFile( const std::string &filename, aiScene *pSce std::vector buffer; TextFileToBuffer( file, buffer ); + pIOHandler->Close( file ); OpenDDLParser myParser; myParser.setBuffer( &buffer[ 0 ], buffer.size() ); From b841ed194bf833dd1f8baf033d54f00f7b2c5095 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:43:49 +0300 Subject: [PATCH 250/305] OpenGEXImporter: Store RefInfo in unique_ptr so they get automatically cleaned up --- code/OpenGEXImporter.cpp | 8 ++++---- code/OpenGEXImporter.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 24dabd1d4..7073a5e76 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -525,7 +525,7 @@ void OpenGEXImporter::handleObjectRefNode( DDLNode *node, aiScene *pScene ) { m_currentNode->mNumMeshes = static_cast(objRefNames.size()); m_currentNode->mMeshes = new unsigned int[ objRefNames.size() ]; if ( !objRefNames.empty() ) { - m_unresolvedRefStack.push_back( new RefInfo( m_currentNode, RefInfo::MeshRef, objRefNames ) ); + m_unresolvedRefStack.push_back( std::unique_ptr( new RefInfo( m_currentNode, RefInfo::MeshRef, objRefNames ) ) ); } } else if ( m_tokenType == Grammar::LightNodeToken ) { // TODO! @@ -544,7 +544,7 @@ void OpenGEXImporter::handleMaterialRefNode( ODDLParser::DDLNode *node, aiScene std::vector matRefNames; getRefNames( node, matRefNames ); if( !matRefNames.empty() ) { - m_unresolvedRefStack.push_back( new RefInfo( m_currentNode, RefInfo::MaterialRef, matRefNames ) ); + m_unresolvedRefStack.push_back( std::unique_ptr( new RefInfo( m_currentNode, RefInfo::MaterialRef, matRefNames ) ) ); } } @@ -1165,8 +1165,8 @@ void OpenGEXImporter::resolveReferences() { } RefInfo *currentRefInfo( nullptr ); - for( std::vector::iterator it = m_unresolvedRefStack.begin(); it != m_unresolvedRefStack.end(); ++it ) { - currentRefInfo = *it; + for( auto it = m_unresolvedRefStack.begin(); it != m_unresolvedRefStack.end(); ++it ) { + currentRefInfo = it->get(); if( nullptr != currentRefInfo ) { aiNode *node( currentRefInfo->m_node ); if( RefInfo::MeshRef == currentRefInfo->m_type ) { diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index 948712869..aefd9ef1d 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -201,7 +201,7 @@ private: std::vector m_cameraCache; std::vector m_lightCache; std::vector m_nodeStack; - std::vector m_unresolvedRefStack; + std::vector > m_unresolvedRefStack; }; } // Namespace OpenGEX From 50b43f76e12e269be8bb7e5985115bc0e5d4d1b7 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Thu, 5 Oct 2017 12:51:08 +0300 Subject: [PATCH 251/305] OpenGEXImporter: Copy materials to scene --- code/OpenGEXImporter.cpp | 14 ++++++++++++++ code/OpenGEXImporter.h | 1 + 2 files changed, 15 insertions(+) diff --git a/code/OpenGEXImporter.cpp b/code/OpenGEXImporter.cpp index 7073a5e76..47e3e3e8c 100644 --- a/code/OpenGEXImporter.cpp +++ b/code/OpenGEXImporter.cpp @@ -322,6 +322,7 @@ void OpenGEXImporter::InternReadFile( const std::string &filename, aiScene *pSce copyMeshes( pScene ); copyCameras( pScene ); copyLights( pScene ); + copyMaterials( pScene ); resolveReferences(); createNodeTree( pScene ); } @@ -1158,6 +1159,19 @@ void OpenGEXImporter::copyLights( aiScene *pScene ) { std::copy( m_lightCache.begin(), m_lightCache.end(), pScene->mLights ); } +//------------------------------------------------------------------------------------------------ +void OpenGEXImporter::copyMaterials( aiScene *pScene ) { + ai_assert( nullptr != pScene ); + + if ( m_materialCache.empty() ) { + return; + } + + pScene->mNumMaterials = static_cast(m_materialCache.size()); + pScene->mMaterials = new aiMaterial*[ pScene->mNumMaterials ]; + std::copy( m_materialCache.begin(), m_materialCache.end(), pScene->mMaterials ); +} + //------------------------------------------------------------------------------------------------ void OpenGEXImporter::resolveReferences() { if( m_unresolvedRefStack.empty() ) { diff --git a/code/OpenGEXImporter.h b/code/OpenGEXImporter.h index aefd9ef1d..c0cde579c 100644 --- a/code/OpenGEXImporter.h +++ b/code/OpenGEXImporter.h @@ -133,6 +133,7 @@ protected: void copyMeshes( aiScene *pScene ); void copyCameras( aiScene *pScene ); void copyLights( aiScene *pScene ); + void copyMaterials( aiScene *pScene ); void resolveReferences(); void pushNode( aiNode *node, aiScene *pScene ); aiNode *popNode(); From 326158633b7d6e495096af81c751ee0ea69a002e Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Fri, 6 Oct 2017 20:32:33 +1100 Subject: [PATCH 252/305] Fixed warnings on MSVC caused by implicit conversions from double to float. --- code/glTF2Asset.inl | 4 ++-- code/glTF2Exporter.cpp | 6 +++--- code/glTFExporter.cpp | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 3082ebfab..8b50fa1d3 100644 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -724,7 +724,7 @@ namespace { SetTextureProperties(r, prop, out); if (Value* scale = FindNumber(*prop, "scale")) { - out.scale = scale->GetDouble(); + out.scale = static_cast(scale->GetDouble()); } } } @@ -735,7 +735,7 @@ namespace { SetTextureProperties(r, prop, out); if (Value* strength = FindNumber(*prop, "strength")) { - out.strength = strength->GetDouble(); + out.strength = static_cast(strength->GetDouble()); } } } diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 4ed8f20ff..8f46f7dc7 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -563,7 +563,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref(jointNamesIndex); vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; jointsPerVertex[vertexId] += 1; @@ -872,7 +872,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmNumPositionKeys / numKeyframes; // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. // Check if we have to cast type here. e.g. uint16_t() - timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; + timeData[i] = static_cast(nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond); } Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); @@ -953,7 +953,7 @@ void glTF2Exporter::ExportAnimations() Ref animRef = mAsset->animations.Create(name); // Parameters - ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, static_cast(anim->mTicksPerSecond)); for (unsigned int j = 0; j < 3; ++j) { std::string channelType; diff --git a/code/glTFExporter.cpp b/code/glTFExporter.cpp index a5df09ac7..92abcc15c 100644 --- a/code/glTFExporter.cpp +++ b/code/glTFExporter.cpp @@ -473,7 +473,7 @@ void ExportSkin(Asset& mAsset, const aiMesh* aimesh, Ref& meshRef, Ref(jointNamesIndex); vertexWeightData[vertexId][jointsPerVertex[vertexId]] = vertWeight; jointsPerVertex[vertexId] += 1; @@ -872,7 +872,7 @@ inline void ExtractAnimationData(Asset& mAsset, std::string& animId, RefmNumPositionKeys / numKeyframes; // mTime is measured in ticks, but GLTF time is measured in seconds, so convert. // Check if we have to cast type here. e.g. uint16_t() - timeData[i] = nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond; + timeData[i] = static_cast(nodeChannel->mPositionKeys[frameIndex].mTime / ticksPerSecond); } Ref timeAccessor = ExportData(mAsset, animId, buffer, static_cast(numKeyframes), &timeData[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_FLOAT); @@ -953,7 +953,7 @@ void glTFExporter::ExportAnimations() Ref animRef = mAsset->animations.Create(name); /******************* Parameters ********************/ - ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, anim->mTicksPerSecond); + ExtractAnimationData(*mAsset, name, animRef, bufferRef, nodeChannel, static_cast(anim->mTicksPerSecond)); for (unsigned int j = 0; j < 3; ++j) { std::string channelType; From 9a79d243f9e61f83ef4ccc89e7532bb06701d244 Mon Sep 17 00:00:00 2001 From: Marco Di Benedetto Date: Sat, 7 Oct 2017 12:36:02 +0200 Subject: [PATCH 253/305] added additional displacement texture token. --- code/ObjFileMtlImporter.cpp | 49 +++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/code/ObjFileMtlImporter.cpp b/code/ObjFileMtlImporter.cpp index 36bb6c2cb..6ef655f8a 100644 --- a/code/ObjFileMtlImporter.cpp +++ b/code/ObjFileMtlImporter.cpp @@ -55,20 +55,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { -// Material specific token -static const std::string DiffuseTexture = "map_Kd"; -static const std::string AmbientTexture = "map_Ka"; -static const std::string SpecularTexture = "map_Ks"; -static const std::string OpacityTexture = "map_d"; -static const std::string EmissiveTexture = "map_emissive"; -static const std::string EmissiveTexture_1 = "map_Ke"; -static const std::string BumpTexture1 = "map_bump"; -static const std::string BumpTexture2 = "map_Bump"; -static const std::string BumpTexture3 = "bump"; -static const std::string NormalTexture = "map_Kn"; -static const std::string ReflectionTexture = "refl"; -static const std::string DisplacementTexture = "disp"; -static const std::string SpecularityTexture = "map_ns"; +// Material specific token (case insensitive compare) +static const std::string DiffuseTexture = "map_Kd"; +static const std::string AmbientTexture = "map_Ka"; +static const std::string SpecularTexture = "map_Ks"; +static const std::string OpacityTexture = "map_d"; +static const std::string EmissiveTexture1 = "map_emissive"; +static const std::string EmissiveTexture2 = "map_Ke"; +static const std::string BumpTexture1 = "map_bump"; +static const std::string BumpTexture2 = "bump"; +static const std::string NormalTexture = "map_Kn"; +static const std::string ReflectionTexture = "refl"; +static const std::string DisplacementTexture1 = "map_disp"; +static const std::string DisplacementTexture2 = "disp"; +static const std::string SpecularityTexture = "map_ns"; // texture option specific token static const std::string BlendUOption = "-blendu"; @@ -329,7 +329,7 @@ void ObjFileMtlImporter::getTexture() { // Ambient texture out = & m_pModel->m_pCurrentMaterial->textureAmbient; clampIndex = ObjFile::Material::TextureAmbientType; - } else if (!ASSIMP_strincmp( pPtr, SpecularTexture.c_str(), static_cast(SpecularTexture.size()) ) ) { + } else if ( !ASSIMP_strincmp( pPtr, SpecularTexture.c_str(), static_cast(SpecularTexture.size()) ) ) { // Specular texture out = & m_pModel->m_pCurrentMaterial->textureSpecular; clampIndex = ObjFile::Material::TextureSpecularType; @@ -337,33 +337,30 @@ void ObjFileMtlImporter::getTexture() { // Opacity texture out = & m_pModel->m_pCurrentMaterial->textureOpacity; clampIndex = ObjFile::Material::TextureOpacityType; - } else if (!ASSIMP_strincmp( pPtr, EmissiveTexture.c_str(), static_cast(EmissiveTexture.size()) ) ) { + } else if ( !ASSIMP_strincmp( pPtr, EmissiveTexture1.c_str(), static_cast(EmissiveTexture1.size()) ) || + !ASSIMP_strincmp( pPtr, EmissiveTexture2.c_str(), static_cast(EmissiveTexture2.size()) ) ) { // Emissive texture out = & m_pModel->m_pCurrentMaterial->textureEmissive; clampIndex = ObjFile::Material::TextureEmissiveType; - } else if ( !ASSIMP_strincmp( pPtr, EmissiveTexture_1.c_str(), static_cast(EmissiveTexture_1.size()) ) ) { - // Emissive texture - out = &m_pModel->m_pCurrentMaterial->textureEmissive; - clampIndex = ObjFile::Material::TextureEmissiveType; } else if ( !ASSIMP_strincmp( pPtr, BumpTexture1.c_str(), static_cast(BumpTexture1.size()) ) || - !ASSIMP_strincmp( pPtr, BumpTexture2.c_str(), static_cast(BumpTexture2.size()) ) || - !ASSIMP_strincmp( pPtr, BumpTexture3.c_str(), static_cast(BumpTexture3.size()) ) ) { + !ASSIMP_strincmp( pPtr, BumpTexture2.c_str(), static_cast(BumpTexture2.size()) ) ) { // Bump texture out = & m_pModel->m_pCurrentMaterial->textureBump; clampIndex = ObjFile::Material::TextureBumpType; - } else if (!ASSIMP_strincmp( pPtr,NormalTexture.c_str(), static_cast(NormalTexture.size()) ) ) { + } else if ( !ASSIMP_strincmp( pPtr,NormalTexture.c_str(), static_cast(NormalTexture.size()) ) ) { // Normal map out = & m_pModel->m_pCurrentMaterial->textureNormal; clampIndex = ObjFile::Material::TextureNormalType; - } else if(!ASSIMP_strincmp( pPtr, ReflectionTexture.c_str(), static_cast(ReflectionTexture.size()) ) ) { + } else if( !ASSIMP_strincmp( pPtr, ReflectionTexture.c_str(), static_cast(ReflectionTexture.size()) ) ) { // Reflection texture(s) //Do nothing here return; - } else if (!ASSIMP_strincmp( pPtr, DisplacementTexture.c_str(), static_cast(DisplacementTexture.size()) ) ) { + } else if ( !ASSIMP_strincmp( pPtr, DisplacementTexture1.c_str(), static_cast(DisplacementTexture1.size()) ) || + !ASSIMP_strincmp( pPtr, DisplacementTexture2.c_str(), static_cast(DisplacementTexture2.size()) ) ) { // Displacement texture out = &m_pModel->m_pCurrentMaterial->textureDisp; clampIndex = ObjFile::Material::TextureDispType; - } else if (!ASSIMP_strincmp( pPtr, SpecularityTexture.c_str(), static_cast(SpecularityTexture.size()) ) ) { + } else if ( !ASSIMP_strincmp( pPtr, SpecularityTexture.c_str(), static_cast(SpecularityTexture.size()) ) ) { // Specularity scaling (glossiness) out = & m_pModel->m_pCurrentMaterial->textureSpecularity; clampIndex = ObjFile::Material::TextureSpecularityType; From 530176825670851b91a27c847e39b82ea4c7b8c8 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 14:33:34 +0300 Subject: [PATCH 254/305] Travis: Remove unused LINUX env variable --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2eab656cc..62ba41606 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,22 +47,22 @@ matrix: include: - os: linux compiler: gcc - env: LINUX=1 TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON ASAN=OFF + env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON ASAN=OFF - os: linux compiler: gcc - env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + env: TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON - os: linux compiler: gcc - env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF - os: linux compiler: gcc - env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF - os: linux compiler: clang - env: LINUX=1 TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + env: TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON - os: linux compiler: clang - env: LINUX=1 SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From 54d14e6e3b92c6b8dbfe9ab0a6002af7a52ac995 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 14:47:52 +0300 Subject: [PATCH 255/305] Travis: Refactor how we build CMake options --- .travis.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.travis.sh b/.travis.sh index 1ab1ee2b1..1c7c4e13e 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,6 +1,32 @@ function generate() { - cmake -G "Unix Makefiles" -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD -DASSIMP_COVERALLS=$ENABLE_COVERALLS -DASSIMP_WERROR=ON -DASSIMP_ASAN=$ASAN + OPTIONS="-DASSIMP_WERROR=ON" + + if [ "$TRAVIS_NO_EXPORT" = "YES" ] ; then + OPTIONS="$OPTIONS -DASSIMP_NO_EXPORT=YES" + else + OPTIONS="$OPTIONS -DASSIMP_NO_EXPORT=NO" + fi + + if [ "$SHARED_BUILD" = "ON" ] ; then + OPTIONS="$OPTIONS -DBUILD_SHARED_LIBS=ON" + else + OPTIONS="$OPTIONS -DBUILD_SHARED_LIBS=OFF" + fi + + if [ "$ENABLE_COVERALLS" = "ON" ] ; then + OPTIONS="$OPTIONS -DASSIMP_COVERALLS=ON" + else + OPTIONS="$OPTIONS -DASSIMP_COVERALLS=OFF" + fi + + if [ "$ASAN" = "ON" ] ; then + OPTIONS="$OPTIONS -DASSIMP_ASAN=ON" + else + OPTIONS="$OPTIONS -DASSIMP_ASAN=OFF" + fi + + cmake -G "Unix Makefiles" $OPTIONS } if [ $ANDROID ]; then From 42142105fa6f4142160d3df42f2200614ed90b76 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 14:59:12 +0300 Subject: [PATCH 256/305] CMake: Be more verbose about enabled options --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0aae76837..c30278b7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,7 +108,10 @@ IF(MSVC) ENDIF(MSVC) IF(NOT BUILD_SHARED_LIBS) + MESSAGE(STATUS "Shared libraries disabled") SET(LINK_SEARCH_START_STATIC TRUE) +ELSE() + MESSAGE(STATUS "Shared libraries enabled") ENDIF(NOT BUILD_SHARED_LIBS) # Define here the needed parameters @@ -209,12 +212,14 @@ ELSEIF( CMAKE_COMPILER_IS_MINGW ) ENDIF() if (ASSIMP_COVERALLS) + MESSAGE(STATUS "Coveralls enabled") INCLUDE(Coveralls) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") endif() if (ASSIMP_WERROR) + MESSAGE(STATUS "Treating warnings as errors") IF (MSVC) add_compile_options(/WX) ELSE() @@ -224,6 +229,7 @@ if (ASSIMP_WERROR) endif() if (ASSIMP_ASAN) + MESSAGE(STATUS "AddressSanitizer enabled") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") endif() From 7037fdfe2c1edb62f10d6609b5f996d083b91841 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:06:46 +0300 Subject: [PATCH 257/305] Travis: ASAN is now implicitly off, disable explicit off --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62ba41606..47d9309b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,22 +47,22 @@ matrix: include: - os: linux compiler: gcc - env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON ASAN=OFF + env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON - os: linux compiler: gcc env: TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON - os: linux compiler: gcc - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - os: linux compiler: gcc - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF - os: linux compiler: clang env: TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON - os: linux compiler: clang - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From ea58801a2dfdb922c3186a14d701c88544846072 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:08:02 +0300 Subject: [PATCH 258/305] Travis: ENABLE_COVERALLS is now implicitly off, disable explicit off --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47d9309b6..e7c02101d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,19 +50,19 @@ matrix: env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON - os: linux compiler: gcc - env: TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + env: TRAVIS_NO_EXPORT=NO ASAN=ON - os: linux compiler: gcc - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO - os: linux compiler: gcc - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO - os: linux compiler: clang - env: TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF ASAN=ON + env: TRAVIS_NO_EXPORT=NO ASAN=ON - os: linux compiler: clang - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO ENABLE_COVERALLS=OFF + env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From f2cf8bf075d29f0abe175bccba39ec8542bbc79b Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:08:41 +0300 Subject: [PATCH 259/305] Travis: Disable GCC AddressSanitizer build, Clang AddressSanitizer is strictly better --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e7c02101d..8787d3765 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,9 +48,6 @@ matrix: - os: linux compiler: gcc env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON - - os: linux - compiler: gcc - env: TRAVIS_NO_EXPORT=NO ASAN=ON - os: linux compiler: gcc env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO From 69b8c1f60d9df235573cee84ee5d551f92e37a9c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:36:54 +0300 Subject: [PATCH 260/305] Travis: TRAVIS_NO_EXPORT is now implicitly off, disable explicit off --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8787d3765..8c9736ef8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,16 +50,16 @@ matrix: env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON - os: linux compiler: gcc - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO + env: SHARED_BUILD=ON - os: linux compiler: gcc - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO + env: SHARED_BUILD=ON - os: linux compiler: clang - env: TRAVIS_NO_EXPORT=NO ASAN=ON + env: ASAN=ON - os: linux compiler: clang - env: SHARED_BUILD=ON TRAVIS_NO_EXPORT=NO + env: SHARED_BUILD=ON install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi From bd65811329b3685674127877fece1fa5e4648c55 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:37:45 +0300 Subject: [PATCH 261/305] Travis: Rename TRAVIS_NO_EXPORT to DISABLE_EXPORTERS so its meaning is more obvious --- .travis.sh | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.sh b/.travis.sh index 1c7c4e13e..f4ef271e4 100755 --- a/.travis.sh +++ b/.travis.sh @@ -2,7 +2,7 @@ function generate() { OPTIONS="-DASSIMP_WERROR=ON" - if [ "$TRAVIS_NO_EXPORT" = "YES" ] ; then + if [ "$DISABLE_EXPORTERS" = "YES" ] ; then OPTIONS="$OPTIONS -DASSIMP_NO_EXPORT=YES" else OPTIONS="$OPTIONS -DASSIMP_NO_EXPORT=NO" diff --git a/.travis.yml b/.travis.yml index 8c9736ef8..b43894525 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ matrix: include: - os: linux compiler: gcc - env: TRAVIS_NO_EXPORT=YES ENABLE_COVERALLS=ON + env: DISABLE_EXPORTERS=YES ENABLE_COVERALLS=ON - os: linux compiler: gcc env: SHARED_BUILD=ON From 684cb88e830c79c434fc472470af847ed43e1d9c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:39:54 +0300 Subject: [PATCH 262/305] Travis: Remove redundant config --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b43894525..5f6d1a233 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,9 +51,6 @@ matrix: - os: linux compiler: gcc env: SHARED_BUILD=ON - - os: linux - compiler: gcc - env: SHARED_BUILD=ON - os: linux compiler: clang env: ASAN=ON From 10b49dfd2543404c7c0b6ceab6287b8ac2b73538 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 15:41:16 +0300 Subject: [PATCH 263/305] Travis: Remove old attempt at disabling default configurations This didn't actually work and now the default configurations are important. --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f6d1a233..3ffa63176 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,14 +36,6 @@ env: - PV=r8e PLATF=linux-x86_64 NDK_HOME=${TRAVIS_BUILD_DIR}/android-ndk-${PV} PATH=${PATH}:${NDK_HOME} matrix: - exclude: - - os: linux - compiler: gcc - env: - - os: linux - compiler: clang - env: - include: - os: linux compiler: gcc From 8f5489243920f568d30c5dd629131fde3b02b651 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 01:27:21 +1100 Subject: [PATCH 264/305] Cleaned up appveyor setup, added VS 2017 to the build matrix and attempted to add running of tests. --- appveyor.yml | 49 ++++++++++++++++++----------- scripts/appveyor/compiler_setup.bat | 36 --------------------- 2 files changed, 30 insertions(+), 55 deletions(-) delete mode 100644 scripts/appveyor/compiler_setup.bat diff --git a/appveyor.yml b/appveyor.yml index b8828710b..5eabc8ada 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,39 +4,50 @@ # clone directory clone_folder: c:\projects\assimp +shallow_clone: true + # branches to build branches: # whitelist only: - master +image: + - Visual Studio 2013 + - Visual Studio 2015 + - Visual Studio 2017 + platform: - - x86 - - x64 + - x86 + - x64 -configuration: - - 14 2015 - - 12 2013 - #- MinGW - #- 10 2010 # only works for x86 +configuration: Release -init: -- if "%platform%" EQU "x64" ( for %%a in (2008 2010 MinGW) do ( if "%Configuration%"=="%%a" (echo "Skipping unsupported configuration" && exit /b 1 ) ) ) +build: + parallel: true + project: Assimp.sln install: -# Make compiler command line tools available -- call c:\projects\assimp\scripts\appveyor\compiler_setup.bat - -build_script: -- cd c:\projects\assimp -- if "%platform%" equ "x64" (cmake CMakeLists.txt -DASSIMP_WERROR=ON -G "Visual Studio %Configuration% Win64") -- if "%platform%" equ "x86" (cmake CMakeLists.txt -DASSIMP_WERROR=ON -G "Visual Studio %Configuration%") -- if "%platform%" equ "x64" (msbuild /m /p:Configuration=Release /p:Platform="x64" Assimp.sln) -- if "%platform%" equ "x86" (msbuild /m /p:Configuration=Release /p:Platform="Win32" Assimp.sln) + - @echo off + - set CMAKE_DEFINES -DASSIMP_WERROR=ON + - if "%platform%"=="x86" set CMAKE_GENERATOR_NAME=%APPVEYOR_BUILD_WORKER_IMAGE% + - if "%platform%"=="x64" set CMAKE_GENERATOR_NAME=%APPVEYOR_BUILD_WORKER_IMAGE% Win64 + - cmake %CMAKE_DEFINES% -G "%CMAKE_GENERATOR_NAME%" +cache: + - code\assimp.dir\%CONFIGURATION% + after_build: - - 7z a assimp.7z c:\projects\assimp\bin\release\* c:\projects\assimp\lib\release\* + - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* + +before_test: + - xcopy /s "%APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\" "%APPVEYOR_BUILD_FOLDER%\test\%CONFIGURATION%\" +test: + assemblies: + only: + - test\%CONFIGURATION%\unit.exe + artifacts: - path: assimp.7z name: assimp_lib diff --git a/scripts/appveyor/compiler_setup.bat b/scripts/appveyor/compiler_setup.bat deleted file mode 100644 index 7d4851493..000000000 --- a/scripts/appveyor/compiler_setup.bat +++ /dev/null @@ -1,36 +0,0 @@ -rem @echo off - -:: Now we declare a scope -Setlocal EnableDelayedExpansion EnableExtensions - -if not defined Configuration set Configuration=14 2015 - -if "%Configuration%"=="MinGW" ( goto :mingw ) - -set arch=x86 - -if "%platform%" EQU "x64" ( set arch=x86_amd64 ) - -if "%Configuration%"=="14 2015" ( - set SET_VS_ENV="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" -) - -if "%Configuration%"=="12 2013" ( - set SET_VS_ENV="C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" -) - -if "%Configuration%"=="11 2012" ( - set SET_VS_ENV="C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat" -) - -if "%Configuration%"=="10 2010" ( - set SET_VS_ENV="C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" -) - -:: Visual Studio detected -endlocal & call %SET_VS_ENV% %arch% -goto :eof - -:: MinGW detected -:mingw -endlocal & set PATH=c:\mingw\bin;%PATH% From 63764ae42a9d0e0229bf02bc0f2b9a720f3f2494 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 01:33:48 +1100 Subject: [PATCH 265/305] Apparently @ escaping batch commands doesn't work in this context. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5eabc8ada..b31137959 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,7 @@ build: project: Assimp.sln install: - - @echo off + - echo off - set CMAKE_DEFINES -DASSIMP_WERROR=ON - if "%platform%"=="x86" set CMAKE_GENERATOR_NAME=%APPVEYOR_BUILD_WORKER_IMAGE% - if "%platform%"=="x64" set CMAKE_GENERATOR_NAME=%APPVEYOR_BUILD_WORKER_IMAGE% Win64 From 4c06abf281ef1eacaec42ca39b5437c7c511f430 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 01:47:08 +1100 Subject: [PATCH 266/305] Replaced the worker image name, which doesn't work as generator name, with a manually created generator name. --- appveyor.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b31137959..653bcf937 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,10 +28,11 @@ build: project: Assimp.sln install: - - echo off - set CMAKE_DEFINES -DASSIMP_WERROR=ON - - if "%platform%"=="x86" set CMAKE_GENERATOR_NAME=%APPVEYOR_BUILD_WORKER_IMAGE% - - if "%platform%"=="x64" set CMAKE_GENERATOR_NAME=%APPVEYOR_BUILD_WORKER_IMAGE% Win64 + - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2013" set CMAKE_GENERATOR_NAME=Visual Studio 12 2013 + - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" set CMAKE_GENERATOR_NAME=Visual Studio 14 2015 + - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" set CMAKE_GENERATOR_NAME=Visual Studio 15 2017 + - if "%platform%"=="x64" set CMAKE_GENERATOR_NAME=%CMAKE_GENERATOR_NAME% Win64 - cmake %CMAKE_DEFINES% -G "%CMAKE_GENERATOR_NAME%" cache: From ba43e3a152f9dc8e7506c383019fdf7347c4369a Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 01:55:10 +1100 Subject: [PATCH 267/305] x86 isn't a valid VS platform. Win32 it is, then. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 653bcf937..d43e8da45 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,7 +18,7 @@ image: - Visual Studio 2017 platform: - - x86 + - Win32 - x64 configuration: Release From d3de8dbf5ffcb430a1624cc9b607e2dcbffaea4a Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 02:03:00 +1100 Subject: [PATCH 268/305] Paths aren't what I expected for the test directory. Trying something else, with a testing call to dir to help track it down. --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d43e8da45..8e35c0e1a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,12 +42,13 @@ after_build: - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* before_test: - - xcopy /s "%APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\" "%APPVEYOR_BUILD_FOLDER%\test\%CONFIGURATION%\" + - dir "%APPVEYOR_BUILD_FOLDER%\bin" + - xcopy /s "%APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%" "%APPVEYOR_BUILD_FOLDER%\bin\test\%CONFIGURATION%" test: assemblies: only: - - test\%CONFIGURATION%\unit.exe + - bin\test\%CONFIGURATION%\unit.exe artifacts: - path: assimp.7z From fe79322959e3f6cdb4ab19f3bcc136a5dced1881 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 02:22:37 +1100 Subject: [PATCH 269/305] Attempting to get the tests detected and run. --- appveyor.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 8e35c0e1a..b461d9dda 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -40,15 +40,11 @@ cache: after_build: - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* - -before_test: - - dir "%APPVEYOR_BUILD_FOLDER%\bin" - - xcopy /s "%APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%" "%APPVEYOR_BUILD_FOLDER%\bin\test\%CONFIGURATION%" test: assemblies: only: - - bin\test\%CONFIGURATION%\unit.exe + - bin\%CONFIGURATION%\unit.exe artifacts: - path: assimp.7z From f925e2cf4e37cf8923bc2348e2c665379675a90e Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 7 Oct 2017 19:08:20 +0200 Subject: [PATCH 270/305] Reproduce issue and remove assertion when a nullptr makes more sence --- code/ObjFileParser.cpp | 8 -------- code/ValidateDataStructure.cpp | 4 +--- test/unit/utObjImportExport.cpp | 21 ++++++++++++--------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/code/ObjFileParser.cpp b/code/ObjFileParser.cpp index 41677fce5..acf275b94 100644 --- a/code/ObjFileParser.cpp +++ b/code/ObjFileParser.cpp @@ -359,8 +359,6 @@ void ObjFileParser::getHomogeneousVector3( std::vector &point3d_arra m_DataIt = skipLine( m_DataIt, m_DataItEnd, m_uiLine ); } -// ------------------------------------------------------------------- -// Get values for two 3D vectors on the same line void ObjFileParser::getTwoVectors3( std::vector &point3d_array_a, std::vector &point3d_array_b ) { ai_real x, y, z; copyNextWord(m_buffer, Buffersize); @@ -388,8 +386,6 @@ void ObjFileParser::getTwoVectors3( std::vector &point3d_array_a, st m_DataIt = skipLine( m_DataIt, m_DataItEnd, m_uiLine ); } -// ------------------------------------------------------------------- -// Get values for a new 2D vector instance void ObjFileParser::getVector2( std::vector &point2d_array ) { ai_real x, y; copyNextWord(m_buffer, Buffersize); @@ -405,8 +401,6 @@ void ObjFileParser::getVector2( std::vector &point2d_array ) { static const std::string DefaultObjName = "defaultobject"; -// ------------------------------------------------------------------- -// Get values for a new face instance void ObjFileParser::getFace( aiPrimitiveType type ) { m_DataIt = getNextToken( m_DataIt, m_DataItEnd ); if ( m_DataIt == m_DataItEnd || *m_DataIt == '\0' ) { @@ -522,8 +516,6 @@ void ObjFileParser::getFace( aiPrimitiveType type ) { m_DataIt = skipLine( m_DataIt, m_DataItEnd, m_uiLine ); } -// ------------------------------------------------------------------- -// Get values for a new material description void ObjFileParser::getMaterialDesc() { // Get next data for material data m_DataIt = getNextToken(m_DataIt, m_DataItEnd); diff --git a/code/ValidateDataStructure.cpp b/code/ValidateDataStructure.cpp index ae1e0d342..f1035e441 100644 --- a/code/ValidateDataStructure.cpp +++ b/code/ValidateDataStructure.cpp @@ -89,9 +89,7 @@ AI_WONT_RETURN void ValidateDSProcess::ReportError(const char* msg,...) ai_assert(iLen > 0); va_end(args); -#ifdef ASSIMP_BUILD_DEBUG - ai_assert( false ); -#endif + throw DeadlyImportError("Validation failed: " + std::string(szBuffer,iLen)); } // ------------------------------------------------------------------------------------------------ diff --git a/test/unit/utObjImportExport.cpp b/test/unit/utObjImportExport.cpp index 94bc71f67..59fd58a6f 100644 --- a/test/unit/utObjImportExport.cpp +++ b/test/unit/utObjImportExport.cpp @@ -46,6 +46,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include +#include using namespace Assimp; @@ -257,15 +259,16 @@ TEST_F( utObjImportExport, issue809_vertex_color_Test ) { TEST_F( utObjImportExport, issue1453_segfault ) { static const std::string ObjModel = - "v 0.0 0.0 0.0" - "v 0.0 0.0 1.0" - "v 0.0 1.0 0.0" - "v 0.0 1.0 1.0" - "v 1.0 0.0 0.0" - "v 1.0 0.0 1.0" - "v 1.0 1.0 0.0" - "v 1.0 1.0 1.0"; + "v 0.0 0.0 0.0\n" + "v 0.0 0.0 1.0\n" + "v 0.0 1.0 0.0\n" + "v 0.0 1.0 1.0\n" + "v 1.0 0.0 0.0\n" + "v 1.0 0.0 1.0\n" + "v 1.0 1.0 0.0\n" + "v 1.0 1.0 1.0\nB"; Assimp::Importer myimporter; - const aiScene* myscene = myimporter.ReadFileFromMemory( ObjModel.c_str(), ObjModel.size(), 0 ); + const aiScene *scene = myimporter.ReadFileFromMemory( ObjModel.c_str(), ObjModel.size(), aiProcess_ValidateDataStructure ); + EXPECT_EQ( nullptr, scene ); } From 26851880e4a098370b9d65d9cad57154c38564a3 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 11:55:54 +1100 Subject: [PATCH 271/305] Attempting to address issues with cloning the repo. shallow_copy seems to fail at times. I'll let it continue to clone the whole history. --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b461d9dda..380a8c577 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,8 +4,6 @@ # clone directory clone_folder: c:\projects\assimp -shallow_clone: true - # branches to build branches: # whitelist From 3e80aabde5cc27b2d02249894423d5edac763dd4 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 12:32:14 +1100 Subject: [PATCH 272/305] Attempting to get tests to run. --- appveyor.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 380a8c577..5a7694a1a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,10 +39,8 @@ cache: after_build: - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* -test: - assemblies: - only: - - bin\%CONFIGURATION%\unit.exe +test_script: + - cmd: bin\%CONFIGURATION%\unit.exe artifacts: - path: assimp.7z From 6a2dfb1efce678d11cf1f6524fabc1ec98ef3a83 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 12:44:54 +1100 Subject: [PATCH 273/305] Fixed up the path to the build artifacts to be cached. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5a7694a1a..6d5a046bf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,7 +34,7 @@ install: - cmake %CMAKE_DEFINES% -G "%CMAKE_GENERATOR_NAME%" cache: - - code\assimp.dir\%CONFIGURATION% + - assimp.dir\%CONFIGURATION% after_build: - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* From 52da099738e3049ecfd85d5051a2d64c61d3a260 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 13:00:27 +1100 Subject: [PATCH 274/305] Updated test output to log to xml and to upload those results appropriately, so they show up in AppVeyor correctly as tests at the end. --- appveyor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 6d5a046bf..26e72b3d0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -40,7 +40,10 @@ after_build: - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe + - cmd: bin\%CONFIGURATION%\unit.exe -gtest_output=xml:testout.xml + +after_test: + - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) artifacts: - path: assimp.7z From 81d3010f73dd86f7831fe4aabde5aa66ac96c3fb Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 16:21:32 +1100 Subject: [PATCH 275/305] Added the mtime_cache script to hopefully improve use of incremental building with appveyor. Reduced the verbosity of the appveyor config. Added use of mtime_cache. Fixed the output of an xml version of the test output for hooking into appveyor. --- appveyor.yml | 29 ++++-- scripts/AppVeyor/cacheglobs.txt | 4 + scripts/AppVeyor/mtime_cache | 177 ++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 scripts/AppVeyor/cacheglobs.txt create mode 100644 scripts/AppVeyor/mtime_cache diff --git a/appveyor.yml b/appveyor.yml index 26e72b3d0..67bf6cdec 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,28 +19,43 @@ platform: - Win32 - x64 +environment: + - build_cache_dir: bin\.mtime_cache + - build_binary_dir: bin\%CONFIGURATION% + - appveyor_script_dir: scripts\AppVeyor + configuration: Release -build: - parallel: true - project: Assimp.sln - install: + - set PATH=C:\Ruby24-x64\bin;%PATH% - set CMAKE_DEFINES -DASSIMP_WERROR=ON - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2013" set CMAKE_GENERATOR_NAME=Visual Studio 12 2013 - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" set CMAKE_GENERATOR_NAME=Visual Studio 14 2015 - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" set CMAKE_GENERATOR_NAME=Visual Studio 15 2017 - if "%platform%"=="x64" set CMAKE_GENERATOR_NAME=%CMAKE_GENERATOR_NAME% Win64 - cmake %CMAKE_DEFINES% -G "%CMAKE_GENERATOR_NAME%" - + cache: - assimp.dir\%CONFIGURATION% + - zlibstatic.dir\%CONFIGURATION% + - zlib.dir\%CONFIGURATION% + - assimp_cmd.dir\%CONFIGURATION% + - assimp_viewer.dir\%CONFIGURATION% + - unit.dir\%CONFIGURATION% + - $(build_cache_dir) + +before_build: + - ruby $(appveyor_script_dir)\mtime_cache -g $(appveyor_script_dir)\cacheglobs.txt -c $(build_cache_dir)\cache.json + +build: + parallel: true + project: Assimp.sln after_build: - - 7z a assimp.7z %APPVEYOR_BUILD_FOLDER%\bin\%CONFIGURATION%\* %APPVEYOR_BUILD_FOLDER%\lib\%CONFIGURATION%\* + - 7z a assimp.7z $(build_binary_dir)\* lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe -gtest_output=xml:testout.xml + - cmd: $(build_binary_dir)\unit.exe --gtest_output=xml:testout.xml after_test: - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) diff --git a/scripts/AppVeyor/cacheglobs.txt b/scripts/AppVeyor/cacheglobs.txt new file mode 100644 index 000000000..0f5f04a4a --- /dev/null +++ b/scripts/AppVeyor/cacheglobs.txt @@ -0,0 +1,4 @@ +code/*.{%{cpp}} +contrib/**/*.{%{cpp}} +include/**/*.{%{cpp}} +test/**/*.{%{cpp}} diff --git a/scripts/AppVeyor/mtime_cache b/scripts/AppVeyor/mtime_cache new file mode 100644 index 000000000..e296e3658 --- /dev/null +++ b/scripts/AppVeyor/mtime_cache @@ -0,0 +1,177 @@ +#!/usr/bin/env ruby + +# +# mtime_cache +# Copyright (c) 2016 Borislav Stanimirov +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +require 'digest/md5' +require 'json' +require 'fileutils' + +VERSION = "1.0.2" + +VERSION_TEXT = "mtime_cache v#{VERSION}" + +USAGE = <] [-g globfile] [-d] [-q|V] [-c cache] +ENDUSAGE + +HELP = < '.mtime_cache.json', :globs => [] } + +ARGV.each do |arg| + case arg + when '-g', '--globfile' then param_arg = :globfile + when '-h', '-?', '--help' then ARGS[:help] = true + when '-v', '--version' then ARGS[:ver] = true + when '-q', '--quiet' then ARGS[:quiet] = true + when '-V', '--verbose' then ARGS[:verbose] = true + when '-d', '--dryrun' then ARGS[:dry] = true + when '-c', '--cache' then param_arg = :cache + else + if param_arg + ARGS[param_arg] = arg + param_arg = nil + else + ARGS[:globs] << arg + end + end +end + +def log(text, level = 0) + return if ARGS[:quiet] + return if level > 0 && !ARGS[:verbose] + puts text +end + +if ARGS[:ver] || ARGS[:help] + log VERSION_TEXT + exit if ARGS[:ver] + log USAGE + log HELP + exit +end + +if ARGS[:globs].empty? && !ARGS[:globfile] + log 'Error: Missing globs' + log USAGE + exit 1 +end + +EXTENSION_PATTERNS = { + :cpp => "c,cc,cpp,cxx,h,hpp,hxx,inl,ipp,inc,ixx" +} + +cache_file = ARGS[:cache] + +cache = {} + +if File.file?(cache_file) + log "Found #{cache_file}" + cache = JSON.parse(File.read(cache_file)) + log "Read #{cache.length} entries" +else + log "#{cache_file} not found. A new one will be created" +end + +globs = ARGS[:globs].map { |g| g % EXTENSION_PATTERNS } + +globfile = ARGS[:globfile] +if globfile + File.open(globfile, 'r').each_line do |line| + line.strip! + next if line.empty? + globs << line % EXTENSION_PATTERNS + end +end + +if globs.empty? + log 'Error: No globs in globfile' + log USAGE + exit 1 +end + +files = {} +num_changed = 0 + +globs.each do |glob| + Dir[glob].each do |file| + next if !File.file?(file) + + mtime = File.mtime(file).to_i + hash = Digest::MD5.hexdigest(File.read(file)) + + cached = cache[file] + + if cached && cached['hash'] == hash && cached['mtime'] < mtime + mtime = cached['mtime'] + + log "mtime_cache: changing mtime of #{file} to #{mtime}", 1 + + File.utime(File.atime(file), Time.at(mtime), file) if !ARGS[:dry] + num_changed += 1 + else + log "mtime_cache: NOT changing mtime of #{file}", 1 + end + + files[file] = { 'mtime' => mtime, 'hash' => hash } + end +end + +log "Changed mtime of #{num_changed} of #{files.length} files" +log "Writing #{cache_file}" + +if !ARGS[:dry] + dirname = File.dirname(cache_file) + unless File.directory?(dirname) + FileUtils.mkdir_p(dirname) + end + File.open(cache_file, 'w').write(JSON.pretty_generate(files)) +end From 90c9884c5e41b5ba8c6bf001710f767e053d6c8b Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 16:24:05 +1100 Subject: [PATCH 276/305] The environment section uses different syntax for a list of vars. --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 67bf6cdec..92d851ed2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,9 +20,9 @@ platform: - x64 environment: - - build_cache_dir: bin\.mtime_cache - - build_binary_dir: bin\%CONFIGURATION% - - appveyor_script_dir: scripts\AppVeyor + build_cache_dir: bin\.mtime_cache + build_binary_dir: bin\%CONFIGURATION% + appveyor_script_dir: scripts\AppVeyor configuration: Release From 58ac747634a9f8b2997c141998272c5c5d5cf935 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 16:28:43 +1100 Subject: [PATCH 277/305] Fixed some bad usage of environment variables. --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 92d851ed2..6bddf4684 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -45,17 +45,17 @@ cache: - $(build_cache_dir) before_build: - - ruby $(appveyor_script_dir)\mtime_cache -g $(appveyor_script_dir)\cacheglobs.txt -c $(build_cache_dir)\cache.json + - ruby %appveyor_script_dir%\mtime_cache -g %appveyor_script_dir%\cacheglobs.txt -c %build_cache_dir%\cache.json build: parallel: true project: Assimp.sln after_build: - - 7z a assimp.7z $(build_binary_dir)\* lib\%CONFIGURATION%\* + - 7z a assimp.7z %build_binary_dir%\* lib\%CONFIGURATION%\* test_script: - - cmd: $(build_binary_dir)\unit.exe --gtest_output=xml:testout.xml + - cmd: %build_binary_dir%\unit.exe --gtest_output=xml:testout.xml after_test: - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) From 5149149b0760501b0aa0731283ae1ca7af0ec54b Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 16:31:46 +1100 Subject: [PATCH 278/305] The environment section doesn't do what I thought it would. Duplication is bad, but seems neccessary. --- appveyor.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6bddf4684..74bb5ce3e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,11 +18,6 @@ image: platform: - Win32 - x64 - -environment: - build_cache_dir: bin\.mtime_cache - build_binary_dir: bin\%CONFIGURATION% - appveyor_script_dir: scripts\AppVeyor configuration: Release @@ -42,20 +37,20 @@ cache: - assimp_cmd.dir\%CONFIGURATION% - assimp_viewer.dir\%CONFIGURATION% - unit.dir\%CONFIGURATION% - - $(build_cache_dir) + - bin\.mtime_cache before_build: - - ruby %appveyor_script_dir%\mtime_cache -g %appveyor_script_dir%\cacheglobs.txt -c %build_cache_dir%\cache.json + - ruby scripts\AppVeyor\mtime_cache -g scripts\AppVeyor\cacheglobs.txt -c bin\.mtime_cache\cache.json build: parallel: true project: Assimp.sln after_build: - - 7z a assimp.7z %build_binary_dir%\* lib\%CONFIGURATION%\* + - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* test_script: - - cmd: %build_binary_dir%\unit.exe --gtest_output=xml:testout.xml + - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml after_test: - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) From 5e5b7f4f75122520c2605a7665ed2335e2c80f7e Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 17:06:23 +1100 Subject: [PATCH 279/305] Refined the appveyor config so that test output is reported even if tests fail. --- appveyor.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 74bb5ce3e..43f83a264 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,10 +51,8 @@ after_build: test_script: - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml - -after_test: - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) - + artifacts: - path: assimp.7z name: assimp_lib From 54d2f25aa597790f47f24e92fdddedcd696aa25a Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 17:46:45 +1100 Subject: [PATCH 280/305] Attempting to get test results reporting even if an error occurrs. --- appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 43f83a264..e3485dd35 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,7 +50,9 @@ after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml + - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || ( +Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml))" && exit 1 +) - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) artifacts: From 022a34e54c275330f3616b1c8bd88966771c8a9f Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 17:50:11 +1100 Subject: [PATCH 281/305] Fixing multiline batch command weirdness. --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e3485dd35..3e958b36c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,8 +51,7 @@ after_build: test_script: - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || ( -Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml))" && exit 1 -) +Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml))" && exit 1) - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) artifacts: From 27b6cc22dbf4120d89a1bfee74b5f1bae1479502 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 17:51:42 +1100 Subject: [PATCH 282/305] No fun/multiline allowed. --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3e958b36c..804dab9d1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,8 +50,7 @@ after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || ( -Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml))" && exit 1) + - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || (Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml))" && exit 1) - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) artifacts: From 61836080c53c64275098756ded69bd680cb23f3c Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 18:01:58 +1100 Subject: [PATCH 283/305] Quote escaping across nested batch and powershell hurts my brain. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 804dab9d1..6fa9a9a13 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,7 +50,7 @@ after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || (Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml))" && exit 1) + - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || (Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile(`\"https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)`\", (Resolve-Path .\testout.xml))" && exit 1) - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) artifacts: From e60e396721fea1a42f030dacc6cba864a4656dae Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 18:42:56 +1100 Subject: [PATCH 284/305] Attempting to address failures with chained batch and powershell commands. --- appveyor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 6fa9a9a13..dee27d967 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,6 +21,9 @@ platform: configuration: Release +matrix: + fast_finish: true + install: - set PATH=C:\Ruby24-x64\bin;%PATH% - set CMAKE_DEFINES -DASSIMP_WERROR=ON @@ -50,7 +53,7 @@ after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || (Powershell.exe -executionpolicy remotesigned -Command "(new-object net.webclient).UploadFile(`\"https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)`\", (Resolve-Path .\testout.xml))" && exit 1) + - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || (Powershell.exe -executionpolicy remotesigned -Command "& '(new-object net.webclient).UploadFile(\"https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)\", (Resolve-Path .\testout.xml))'" && exit 1) - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) artifacts: From d8e3952b64d4de3f0353fdb72c62a18b4c6cce10 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 18:57:35 +1100 Subject: [PATCH 285/305] Merging the test_script with test reporting was a terrible idea in retrospect. on_finish should serve the purpose. --- appveyor.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index dee27d967..f008b3cf2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -53,9 +53,11 @@ after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* test_script: - - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml || (Powershell.exe -executionpolicy remotesigned -Command "& '(new-object net.webclient).UploadFile(\"https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)\", (Resolve-Path .\testout.xml))'" && exit 1) - - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) + - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml +on_finish: + - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testout.xml)) + artifacts: - path: assimp.7z name: assimp_lib From 3b4ffbc1b694af759b76ce6dda0187cf7a1dda56 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 19:16:03 +1100 Subject: [PATCH 286/305] Test failures are now getting properly reported. Turning off 'fast finish' to allow all tests to execute. --- appveyor.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f008b3cf2..c343c0d21 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,9 +21,6 @@ platform: configuration: Release -matrix: - fast_finish: true - install: - set PATH=C:\Ruby24-x64\bin;%PATH% - set CMAKE_DEFINES -DASSIMP_WERROR=ON From 7e033c6cefe66b28a8cbb921130759505dec875c Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sun, 8 Oct 2017 12:45:02 +0300 Subject: [PATCH 287/305] FBX: Remove useless const qualifier from return value const qualifier on primitive return type does nothing. --- code/FBXDocument.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/FBXDocument.h b/code/FBXDocument.h index af89f53d5..a4e28b2aa 100644 --- a/code/FBXDocument.h +++ b/code/FBXDocument.h @@ -596,10 +596,10 @@ public: return textures[index]; } - const int textureCount() const { + int textureCount() const { return static_cast(textures.size()); } - const BlendMode GetBlendMode() const + BlendMode GetBlendMode() const { return blendMode; } @@ -647,7 +647,7 @@ public: return content; } - const uint32_t ContentLength() const { + uint32_t ContentLength() const { return contentLength; } From cd64eae59058ab00d3f031730c823ccb53fd5a29 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sun, 8 Oct 2017 12:45:48 +0300 Subject: [PATCH 288/305] GenericProperty: Remove useless const qualifier from return value --- code/GenericProperty.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/GenericProperty.h b/code/GenericProperty.h index 8632e7577..454f4952b 100644 --- a/code/GenericProperty.h +++ b/code/GenericProperty.h @@ -111,7 +111,7 @@ inline void SetGenericPropertyPtr(std::map< unsigned int, T* >& list, // ------------------------------------------------------------------------------------------------ template -inline const bool HasGenericProperty(const std::map< unsigned int, T >& list, +inline bool HasGenericProperty(const std::map< unsigned int, T >& list, const char* szName) { ai_assert(NULL != szName); From f90d874978f2fb0cafde4fff74225319a5dce823 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sun, 8 Oct 2017 12:48:45 +0300 Subject: [PATCH 289/305] Open3DGC: Remove redundant const qualifiers from return types --- contrib/Open3DGC/o3dgcAdjacencyInfo.h | 4 +-- contrib/Open3DGC/o3dgcBinaryStream.h | 6 ++-- contrib/Open3DGC/o3dgcDynamicVector.h | 8 +++--- contrib/Open3DGC/o3dgcFIFO.h | 4 +-- contrib/Open3DGC/o3dgcIndexedFaceSet.h | 32 ++++++++++----------- contrib/Open3DGC/o3dgcTriangleListEncoder.h | 8 +++--- contrib/Open3DGC/o3dgcVector.h | 4 +-- 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/contrib/Open3DGC/o3dgcAdjacencyInfo.h b/contrib/Open3DGC/o3dgcAdjacencyInfo.h index 6b53a242d..72fe3d4c6 100644 --- a/contrib/Open3DGC/o3dgcAdjacencyInfo.h +++ b/contrib/Open3DGC/o3dgcAdjacencyInfo.h @@ -140,8 +140,8 @@ namespace o3dgc { return End(element) - Begin(element); } - long * const GetNumNeighborsBuffer() { return m_numNeighbors;} - long * const GetNeighborsBuffer() { return m_neighbors;} + long * GetNumNeighborsBuffer() { return m_numNeighbors;} + long * GetNeighborsBuffer() { return m_neighbors;} private: long m_neighborsSize; // actual allocated size for m_neighbors diff --git a/contrib/Open3DGC/o3dgcBinaryStream.h b/contrib/Open3DGC/o3dgcBinaryStream.h index 9f4aefe55..19e3df973 100644 --- a/contrib/Open3DGC/o3dgcBinaryStream.h +++ b/contrib/Open3DGC/o3dgcBinaryStream.h @@ -395,15 +395,15 @@ namespace o3dgc { return m_stream.GetSize(); } - const unsigned char * const GetBuffer(unsigned long position) const + const unsigned char * GetBuffer(unsigned long position) const { return m_stream.GetBuffer() + position; } - unsigned char * const GetBuffer(unsigned long position) + unsigned char * GetBuffer(unsigned long position) { return (m_stream.GetBuffer() + position); } - unsigned char * const GetBuffer() + unsigned char * GetBuffer() { return m_stream.GetBuffer(); } diff --git a/contrib/Open3DGC/o3dgcDynamicVector.h b/contrib/Open3DGC/o3dgcDynamicVector.h index edc97d83c..aa7fb3142 100644 --- a/contrib/Open3DGC/o3dgcDynamicVector.h +++ b/contrib/Open3DGC/o3dgcDynamicVector.h @@ -48,10 +48,10 @@ namespace o3dgc unsigned long GetNVector() const { return m_num;} unsigned long GetDimVector() const { return m_dim;} unsigned long GetStride() const { return m_stride;} - const Real * const GetMin() const { return m_min;} - const Real * const GetMax() const { return m_max;} - const Real * const GetVectors() const { return m_vectors;} - Real * const GetVectors() { return m_vectors;} + const Real * GetMin() const { return m_min;} + const Real * GetMax() const { return m_max;} + const Real * GetVectors() const { return m_vectors;} + Real * GetVectors() { return m_vectors;} Real GetMin(unsigned long j) const { return m_min[j];} Real GetMax(unsigned long j) const { return m_max[j];} diff --git a/contrib/Open3DGC/o3dgcFIFO.h b/contrib/Open3DGC/o3dgcFIFO.h index 874c26475..4a5555f2a 100644 --- a/contrib/Open3DGC/o3dgcFIFO.h +++ b/contrib/Open3DGC/o3dgcFIFO.h @@ -81,8 +81,8 @@ namespace o3dgc m_end = 0; } } - const unsigned long GetSize() const { return m_size;}; - const unsigned long GetAllocatedSize() const { return m_allocated;}; + unsigned long GetSize() const { return m_size;}; + unsigned long GetAllocatedSize() const { return m_allocated;}; void Clear() { m_start = m_end = m_size = 0;}; private: diff --git a/contrib/Open3DGC/o3dgcIndexedFaceSet.h b/contrib/Open3DGC/o3dgcIndexedFaceSet.h index 4af9de437..adb8cb001 100644 --- a/contrib/Open3DGC/o3dgcIndexedFaceSet.h +++ b/contrib/Open3DGC/o3dgcIndexedFaceSet.h @@ -62,26 +62,26 @@ namespace o3dgc } unsigned long GetNumFloatAttributes() const { return m_numFloatAttributes;} unsigned long GetNumIntAttributes() const { return m_numIntAttributes ;} - const Real * const GetCoordMin () const { return m_coordMin;} - const Real * const GetCoordMax () const { return m_coordMax;} - const Real * const GetNormalMin () const { return m_normalMin;} - const Real * const GetNormalMax () const { return m_normalMax;} + const Real * GetCoordMin () const { return m_coordMin;} + const Real * GetCoordMax () const { return m_coordMax;} + const Real * GetNormalMin () const { return m_normalMin;} + const Real * GetNormalMax () const { return m_normalMax;} Real GetCoordMin (int j) const { return m_coordMin[j] ;} Real GetCoordMax (int j) const { return m_coordMax[j] ;} Real GetNormalMin (int j) const { return m_normalMin[j] ;} Real GetNormalMax (int j) const { return m_normalMax[j] ;} - const O3DGCIFSFloatAttributeType GetFloatAttributeType(unsigned long a) const + O3DGCIFSFloatAttributeType GetFloatAttributeType(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_FLOAT_ATTRIBUTES); return m_typeFloatAttribute[a]; } - const O3DGCIFSIntAttributeType GetIntAttributeType(unsigned long a) const + O3DGCIFSIntAttributeType GetIntAttributeType(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_INT_ATTRIBUTES); return m_typeIntAttribute[a]; } - const unsigned long GetFloatAttributeDim(unsigned long a) const + unsigned long GetFloatAttributeDim(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_FLOAT_ATTRIBUTES); return m_dimFloatAttribute[a]; @@ -91,12 +91,12 @@ namespace o3dgc assert(a < O3DGC_SC3DMC_MAX_NUM_INT_ATTRIBUTES); return m_dimIntAttribute[a]; } - const Real * const GetFloatAttributeMin(unsigned long a) const + const Real * GetFloatAttributeMin(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_FLOAT_ATTRIBUTES); return &(m_minFloatAttribute[a * O3DGC_SC3DMC_MAX_DIM_ATTRIBUTES]); } - const Real * const GetFloatAttributeMax(unsigned long a) const + const Real * GetFloatAttributeMax(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_FLOAT_ATTRIBUTES); return &(m_maxFloatAttribute[a * O3DGC_SC3DMC_MAX_DIM_ATTRIBUTES]); @@ -118,17 +118,17 @@ namespace o3dgc bool GetSolid() const { return m_solid ;} bool GetConvex() const { return m_convex ;} bool GetIsTriangularMesh() const { return m_isTriangularMesh;} - const unsigned long * const GetIndexBufferID() const { return m_indexBufferID ;} - const T * const GetCoordIndex() const { return m_coordIndex;} - T * const GetCoordIndex() { return m_coordIndex;} - Real * const GetCoord() const { return m_coord ;} - Real * const GetNormal() const { return m_normal ;} - Real * const GetFloatAttribute(unsigned long a) const + const unsigned long * GetIndexBufferID() const { return m_indexBufferID ;} + const T * GetCoordIndex() const { return m_coordIndex;} + T * GetCoordIndex() { return m_coordIndex;} + Real * GetCoord() const { return m_coord ;} + Real * GetNormal() const { return m_normal ;} + Real * GetFloatAttribute(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_FLOAT_ATTRIBUTES); return m_floatAttribute[a]; } - long * const GetIntAttribute(unsigned long a) const + long * GetIntAttribute(unsigned long a) const { assert(a < O3DGC_SC3DMC_MAX_NUM_INT_ATTRIBUTES); return m_intAttribute[a] ; diff --git a/contrib/Open3DGC/o3dgcTriangleListEncoder.h b/contrib/Open3DGC/o3dgcTriangleListEncoder.h index cf790ecc3..c09172273 100644 --- a/contrib/Open3DGC/o3dgcTriangleListEncoder.h +++ b/contrib/Open3DGC/o3dgcTriangleListEncoder.h @@ -50,10 +50,10 @@ namespace o3dgc BinaryStream & bstream); O3DGCStreamType GetStreamType() const { return m_streamType; } void SetStreamType(O3DGCStreamType streamType) { m_streamType = streamType; } - const long * const GetInvVMap() const { return m_invVMap;} - const long * const GetInvTMap() const { return m_invTMap;} - const long * const GetVMap() const { return m_vmap;} - const long * const GetTMap() const { return m_tmap;} + const long * GetInvVMap() const { return m_invVMap;} + const long * GetInvTMap() const { return m_invTMap;} + const long * GetVMap() const { return m_vmap;} + const long * GetTMap() const { return m_tmap;} const AdjacencyInfo & GetVertexToTriangle() const { return m_vertexToTriangle;} private: diff --git a/contrib/Open3DGC/o3dgcVector.h b/contrib/Open3DGC/o3dgcVector.h index e766e2b7f..08d3ed564 100644 --- a/contrib/Open3DGC/o3dgcVector.h +++ b/contrib/Open3DGC/o3dgcVector.h @@ -88,8 +88,8 @@ namespace o3dgc assert(m_size < m_allocated); m_buffer[m_size++] = value; } - const T * const GetBuffer() const { return m_buffer;}; - T * const GetBuffer() { return m_buffer;}; + const T * GetBuffer() const { return m_buffer;}; + T * GetBuffer() { return m_buffer;}; unsigned long GetSize() const { return m_size;}; void SetSize(unsigned long size) { From 4a915653f5949b2aa173d2bc3466cf84fa92db48 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sun, 8 Oct 2017 23:42:28 +1100 Subject: [PATCH 290/305] Fixed IOStream reporting a file size of 0 for files that have been written, but not yet been flushed to disk. --- test/unit/utDefaultIOStream.cpp | 5 +++++ test/unit/utIOStreamBuffer.cpp | 13 +++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/unit/utDefaultIOStream.cpp b/test/unit/utDefaultIOStream.cpp index c0d82b6dc..128e4e6ef 100644 --- a/test/unit/utDefaultIOStream.cpp +++ b/test/unit/utDefaultIOStream.cpp @@ -69,6 +69,11 @@ TEST_F( utDefaultIOStream, FileSizeTest ) { auto vflush = std::fflush( fs ); ASSERT_EQ(vflush, 0); + std::fclose(fs); + fs = std::fopen(fpath, "r"); + + ASSERT_NE(nullptr, fs); + TestDefaultIOStream myStream( fs, fpath); size_t size = myStream.FileSize(); EXPECT_EQ( size, dataSize); diff --git a/test/unit/utIOStreamBuffer.cpp b/test/unit/utIOStreamBuffer.cpp index a1b67da44..f53a9c9d5 100644 --- a/test/unit/utIOStreamBuffer.cpp +++ b/test/unit/utIOStreamBuffer.cpp @@ -90,7 +90,11 @@ TEST_F( IOStreamBufferTest, open_close_Test ) { auto written = std::fwrite( data, sizeof(*data), dataCount, fs ); EXPECT_NE( 0U, written ); - std::fflush( fs ); + auto flushResult = std::fflush( fs ); + ASSERT_EQ(0, flushResult); + std::fclose( fs ); + fs = std::fopen(fname, "r"); + ASSERT_NE(nullptr, fs); { TestDefaultIOStream myStream( fs, fname ); @@ -112,7 +116,12 @@ TEST_F( IOStreamBufferTest, readlineTest ) { auto written = std::fwrite( data, sizeof(*data), dataCount, fs ); EXPECT_NE( 0U, written ); - std::fflush( fs ); + + auto flushResult = std::fflush(fs); + ASSERT_EQ(0, flushResult); + std::fclose(fs); + fs = std::fopen(fname, "r"); + ASSERT_NE(nullptr, fs); const auto tCacheSize = 26u; From b8ad03baa1f96b83357cc5f14de854cd0f079a0b Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 10 Oct 2017 19:45:57 +1100 Subject: [PATCH 291/305] Because I have to change a file somewhere to properly test the cache on AppVeyor, I've made some whitespace a bit better. --- code/LogAux.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/LogAux.h b/code/LogAux.h index 08b1b3c17..a46b9c6ad 100644 --- a/code/LogAux.h +++ b/code/LogAux.h @@ -60,34 +60,34 @@ public: // ------------------------------------------------------------------------------------------------ static void ThrowException(const std::string& msg) { - throw DeadlyImportError(Prefix()+msg); + throw DeadlyImportError(Prefix() + msg); } // ------------------------------------------------------------------------------------------------ static void LogWarn(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->warn(Prefix() +(std::string)message); + DefaultLogger::get()->warn(Prefix() + (std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogError(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->error(Prefix() +(std::string)message); + DefaultLogger::get()->error(Prefix() + (std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogInfo(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->info(Prefix() +(std::string)message); + DefaultLogger::get()->info(Prefix() + (std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogDebug(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->debug(Prefix() +(std::string)message); + DefaultLogger::get()->debug(Prefix() + (std::string)message); } } From bb173749c191412723c41870ba5c812124988e87 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 10 Oct 2017 20:11:21 +1100 Subject: [PATCH 292/305] Attempting to fix the directories being cached. --- appveyor.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c343c0d21..03b0ce73f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,6 +10,9 @@ branches: only: - master +matrix: + fast_finish: true + image: - Visual Studio 2013 - Visual Studio 2015 @@ -31,12 +34,12 @@ install: - cmake %CMAKE_DEFINES% -G "%CMAKE_GENERATOR_NAME%" cache: - - assimp.dir\%CONFIGURATION% - - zlibstatic.dir\%CONFIGURATION% - - zlib.dir\%CONFIGURATION% - - assimp_cmd.dir\%CONFIGURATION% - - assimp_viewer.dir\%CONFIGURATION% - - unit.dir\%CONFIGURATION% + - bin\assimp.dir\%CONFIGURATION% + - bin\zlibstatic.dir\%CONFIGURATION% + - bin\zlib.dir\%CONFIGURATION% + - bin\assimp_cmd.dir\%CONFIGURATION% + - bin\assimp_viewer.dir\%CONFIGURATION% + - bin\unit.dir\%CONFIGURATION% - bin\.mtime_cache before_build: @@ -48,6 +51,9 @@ build: after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* + - cmd: dir + - cmd: dir bin\ + - cmd: exit 1 test_script: - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml From dce2be9e09cd8fd14a763f64a345515fdea4800a Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 10 Oct 2017 21:24:40 +1100 Subject: [PATCH 293/305] I think I've worked out where the obj's are located. We shall see if the cache picks them up. --- appveyor.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 03b0ce73f..1b87286f6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,12 +34,12 @@ install: - cmake %CMAKE_DEFINES% -G "%CMAKE_GENERATOR_NAME%" cache: - - bin\assimp.dir\%CONFIGURATION% - - bin\zlibstatic.dir\%CONFIGURATION% - - bin\zlib.dir\%CONFIGURATION% - - bin\assimp_cmd.dir\%CONFIGURATION% - - bin\assimp_viewer.dir\%CONFIGURATION% - - bin\unit.dir\%CONFIGURATION% + - code\assimp.dir\%CONFIGURATION% + - contrib\zlib\zlibstatic.dir\%CONFIGURATION% + - contrib\zlib\zlib.dir\%CONFIGURATION% + - tools\assimp_cmd\assimp_cmd.dir\%CONFIGURATION% + - tools\assimp_view\assimp_viewer.dir\%CONFIGURATION% + - test\unit.dir\%CONFIGURATION% - bin\.mtime_cache before_build: @@ -51,9 +51,6 @@ build: after_build: - 7z a assimp.7z bin\%CONFIGURATION%\* lib\%CONFIGURATION%\* - - cmd: dir - - cmd: dir bin\ - - cmd: exit 1 test_script: - cmd: bin\%CONFIGURATION%\unit.exe --gtest_output=xml:testout.xml From 1497cc27b2769efe0e74a5c0e206a2a32fb43c58 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 10 Oct 2017 22:08:13 +1100 Subject: [PATCH 294/305] Cleaned up the whitespace again. Let's see how the cache does. --- code/LogAux.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/LogAux.h b/code/LogAux.h index a46b9c6ad..f2bc14d54 100644 --- a/code/LogAux.h +++ b/code/LogAux.h @@ -60,34 +60,34 @@ public: // ------------------------------------------------------------------------------------------------ static void ThrowException(const std::string& msg) { - throw DeadlyImportError(Prefix() + msg); + throw DeadlyImportError(Prefix()+msg); } // ------------------------------------------------------------------------------------------------ static void LogWarn(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->warn(Prefix() + (std::string)message); + DefaultLogger::get()->warn(Prefix()+(std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogError(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->error(Prefix() + (std::string)message); + DefaultLogger::get()->error(Prefix()+(std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogInfo(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->info(Prefix() + (std::string)message); + DefaultLogger::get()->info(Prefix()+(std::string)message); } } // ------------------------------------------------------------------------------------------------ static void LogDebug(const Formatter::format& message) { if (!DefaultLogger::isNullLogger()) { - DefaultLogger::get()->debug(Prefix() + (std::string)message); + DefaultLogger::get()->debug(Prefix()+(std::string)message); } } From fa91a0f64c16f9f2afcf6b631302bf918527e2f7 Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Tue, 10 Oct 2017 22:45:45 +1100 Subject: [PATCH 295/305] Another minor source change, this time even more minor than the last. Let's see what the cache can do with this. --- code/PretransformVertices.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/PretransformVertices.cpp b/code/PretransformVertices.cpp index 5fc294618..0751c4556 100644 --- a/code/PretransformVertices.cpp +++ b/code/PretransformVertices.cpp @@ -104,7 +104,7 @@ unsigned int PretransformVertices::CountNodes( aiNode* pcNode ) // ------------------------------------------------------------------------------------------------ // Get a bitwise combination identifying the vertex format of a mesh -unsigned int PretransformVertices::GetMeshVFormat(aiMesh* pcMesh) +unsigned int PretransformVertices::GetMeshVFormat( aiMesh* pcMesh ) { // the vertex format is stored in aiMesh::mBones for later retrieval. // there isn't a good reason to compute it a few hundred times From 8f141c1966121f9b57756d8c40bc73d0f94c8501 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 10 Oct 2017 18:56:37 +0200 Subject: [PATCH 296/305] Update utObjImportExport.cpp Remove unused header. --- test/unit/utObjImportExport.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/utObjImportExport.cpp b/test/unit/utObjImportExport.cpp index 2fc4d0a31..4aafa3dc2 100644 --- a/test/unit/utObjImportExport.cpp +++ b/test/unit/utObjImportExport.cpp @@ -47,7 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include using namespace Assimp; From 0229a3acf359ccc1b8ee597a1f92541ba38cbcbe Mon Sep 17 00:00:00 2001 From: Jesper Smith Date: Thu, 27 Jul 2017 17:42:01 -0500 Subject: [PATCH 297/305] Added supported for custom IO Systems in Java. Implemented ClassLoader IO System --- port/jassimp/jassimp-native/src/jassimp.cpp | 258 +++++++++++++++++- port/jassimp/jassimp-native/src/jassimp.h | 2 +- .../src/jassimp/AiClassLoaderIOSystem.java | 130 +++++++++ .../jassimp/src/jassimp/AiIOStream.java | 55 ++++ .../jassimp/src/jassimp/AiIOSystem.java | 54 ++++ .../src/jassimp/AiInputStreamIOStream.java | 102 +++++++ port/jassimp/jassimp/src/jassimp/Jassimp.java | 36 ++- 7 files changed, 619 insertions(+), 18 deletions(-) create mode 100644 port/jassimp/jassimp/src/jassimp/AiClassLoaderIOSystem.java create mode 100644 port/jassimp/jassimp/src/jassimp/AiIOStream.java create mode 100644 port/jassimp/jassimp/src/jassimp/AiIOSystem.java create mode 100644 port/jassimp/jassimp/src/jassimp/AiInputStreamIOStream.java diff --git a/port/jassimp/jassimp-native/src/jassimp.cpp b/port/jassimp/jassimp-native/src/jassimp.cpp index 226e416aa..475f6c5a0 100644 --- a/port/jassimp/jassimp-native/src/jassimp.cpp +++ b/port/jassimp/jassimp-native/src/jassimp.cpp @@ -1,7 +1,9 @@ #include "jassimp.h" -#include +#include #include +#include +#include #ifdef JNI_LOG @@ -12,9 +14,11 @@ #define lprintf(...) printf (__VA_ARGS__) #endif /* ANDROID */ #else -#define lprintf +#define lprintf #endif +static std::string gLastErrorString; + // Automatically deletes a local ref when it goes out of scope class SmartLocalRef { private: @@ -270,6 +274,81 @@ static bool callv(JNIEnv *env, jobject object, const char* typeName, return true; } +static jobject callo(JNIEnv *env, jobject object, const char* typeName, const char* methodName, + const char* signature,/* const*/ jvalue* params) +{ + jclass clazz = env->FindClass(typeName); + SmartLocalRef clazzRef(env, clazz); + + if (NULL == clazz) + { + lprintf("could not find class %s\n", typeName); + return NULL; + } + + jmethodID mid = env->GetMethodID(clazz, methodName, signature); + + if (NULL == mid) + { + lprintf("could not find method %s with signature %s in type %s\n", methodName, signature, typeName); + return NULL; + } + + jobject jReturnValue = env->CallObjectMethodA(object, mid, params); + + return jReturnValue; +} + +static int calli(JNIEnv *env, jobject object, const char* typeName, const char* methodName, + const char* signature) +{ + jclass clazz = env->FindClass(typeName); + SmartLocalRef clazzRef(env, clazz); + + if (NULL == clazz) + { + lprintf("could not find class %s\n", typeName); + return false; + } + + jmethodID mid = env->GetMethodID(clazz, methodName, signature); + + if (NULL == mid) + { + lprintf("could not find method %s with signature %s in type %s\n", methodName, signature, typeName); + return false; + } + + jint jReturnValue = env->CallIntMethod(object, mid); + + return (int) jReturnValue; +} + +static int callc(JNIEnv *env, jobject object, const char* typeName, const char* methodName, + const char* signature) +{ + jclass clazz = env->FindClass(typeName); + SmartLocalRef clazzRef(env, clazz); + + if (NULL == clazz) + { + lprintf("could not find class %s\n", typeName); + return false; + } + + jmethodID mid = env->GetMethodID(clazz, methodName, signature); + + if (NULL == mid) + { + lprintf("could not find method %s with signature %s in type %s\n", methodName, signature, typeName); + return false; + } + + jint jReturnValue = env->CallCharMethod(object, mid); + + return (int) jReturnValue; +} + static bool callStaticObject(JNIEnv *env, const char* typeName, const char* methodName, const char* signature,/* const*/ jvalue* params, jobject& returnValue) @@ -359,6 +438,155 @@ static bool copyBufferArray(JNIEnv *env, jobject jMesh, const char* jBufferName, return true; } +class JavaIOStream : public Assimp::IOStream +{ +private: + size_t pos; + size_t size; + char* buffer; + jobject jIOStream; + + +public: + JavaIOStream(size_t size, char* buffer, jobject jIOStream) : + pos(0), + size(size), + buffer(buffer), + jIOStream(jIOStream) + {}; + + + ~JavaIOStream(void) + { + free(buffer); + }; + + size_t Read(void* pvBuffer, size_t pSize, size_t pCount) + { + const size_t cnt = std::min(pCount,(size - pos)/pSize); + const size_t ofs = pSize*cnt; + + memcpy(pvBuffer, buffer + pos, ofs); + pos += ofs; + + return cnt; + }; + size_t Write(const void* pvBuffer, size_t pSize, size_t pCount) {}; + + aiReturn Seek(size_t pOffset, aiOrigin pOrigin) + { + if (aiOrigin_SET == pOrigin) { + if (pOffset >= size) { + return AI_FAILURE; + } + pos = pOffset; + } + else if (aiOrigin_END == pOrigin) { + if (pOffset >= size) { + return AI_FAILURE; + } + pos = size-pOffset; + } + else { + if (pOffset + pos >= size) { + return AI_FAILURE; + } + pos += pOffset; + } + return AI_SUCCESS; + }; + + size_t Tell(void) const + { + return pos; + }; + + size_t FileSize() const + { + return size; + }; + + void Flush() {}; + + + jobject javaObject() + { + return jIOStream; + }; + + +}; + + +class JavaIOSystem : public Assimp::IOSystem { + private: + JNIEnv* mJniEnv; + jobject& mJavaIOSystem; + + public: + JavaIOSystem(JNIEnv* env, jobject& javaIOSystem) : + mJniEnv(env), + mJavaIOSystem(javaIOSystem) + {}; + + bool Exists( const char* pFile) const + { + jvalue params[1]; + params[0].l = mJniEnv->NewStringUTF(pFile); + return call(mJniEnv, mJavaIOSystem, "jassimp/AiIOSystem", "exists", "(Ljava/lang/String;)Z", params); + + }; + char getOsSeparator() const + { + return (char) callc(mJniEnv, mJavaIOSystem, "jassimp/AiIOSystem", "getOsSeparator", "()C"); + }; + + Assimp::IOStream* Open(const char* pFile,const char* pMode = "rb") + { + jvalue params[2]; + params[0].l = mJniEnv->NewStringUTF(pFile); + params[1].l = mJniEnv->NewStringUTF(pMode); + + + jobject jStream = callo(mJniEnv, mJavaIOSystem, "jassimp/AiIOSystem", "open", "(Ljava/lang/String;Ljava/lang/String;)Ljassimp/AiIOStream;", params); + if(NULL == jStream) + { + lprintf("NULL object from AiIOSystem.open\n"); + return NULL; + } + + size_t size = calli(mJniEnv, jStream, "jassimp/AiIOStream", "getFileSize", "()I"); + lprintf("Model file size is %d\n", size); + + char* buffer = (char*)malloc(size); + jobject javaBuffer = mJniEnv->NewDirectByteBuffer(buffer, size); + + jvalue readParams[1]; + readParams[0].l = javaBuffer; + if(call(mJniEnv, jStream, "jassimp/AiIOStream", "read", "(Ljava/nio/ByteBuffer;)Z", readParams)) + { + return new JavaIOStream(size, buffer, jStream); + } + else + { + lprintf("Read failure on AiIOStream.read"); + free(buffer); + return NULL; + } + + }; + void Close( Assimp::IOStream* pFile) + { + + jvalue params[1]; + params[0].l = ((JavaIOStream*) pFile)->javaObject(); + callv(mJniEnv, mJavaIOSystem, "jassimp/AiIOSystem", "close", "(Ljassimp/AiIOStream;)V", params); + delete pFile; + }; + + + +}; static bool loadMeshes(JNIEnv *env, const aiScene* cScene, jobject& jScene) @@ -1474,7 +1702,7 @@ JNIEXPORT jint JNICALL Java_jassimp_Jassimp_getlongsize JNIEXPORT jstring JNICALL Java_jassimp_Jassimp_getErrorString (JNIEnv *env, jclass jClazz) { - const char *err = aiGetErrorString(); + const char *err = gLastErrorString.c_str(); if (NULL == err) { @@ -1486,18 +1714,26 @@ JNIEXPORT jstring JNICALL Java_jassimp_Jassimp_getErrorString JNIEXPORT jobject JNICALL Java_jassimp_Jassimp_aiImportFile - (JNIEnv *env, jclass jClazz, jstring jFilename, jlong postProcess) + (JNIEnv *env, jclass jClazz, jstring jFilename, jlong postProcess, jobject ioSystem) { jobject jScene = NULL; /* convert params */ const char* cFilename = env->GetStringUTFChars(jFilename, NULL); + + Assimp::Importer imp; - + + if(ioSystem != NULL) + { + imp.SetIOHandler(new JavaIOSystem(env, ioSystem)); + lprintf("Created aiFileIO\n"); + } + lprintf("opening file: %s\n", cFilename); /* do import */ - const aiScene *cScene = aiImportFile(cFilename, (unsigned int) postProcess); + const aiScene *cScene = imp.ReadFile(cFilename, (unsigned int) postProcess); if (!cScene) { @@ -1552,19 +1788,13 @@ error: /* thats really a problem because we cannot throw in this case */ env->FatalError("could not throw java.io.IOException"); } - - env->ThrowNew(exception, aiGetErrorString()); + gLastErrorString = imp.GetErrorString(); + env->ThrowNew(exception, gLastErrorString.c_str()); lprintf("problem detected\n"); } end: - /* - * NOTE: this releases all memory used in the native domain. - * Ensure all data has been passed to java before! - */ - aiReleaseImport(cScene); - /* free params */ env->ReleaseStringUTFChars(jFilename, cFilename); diff --git a/port/jassimp/jassimp-native/src/jassimp.h b/port/jassimp/jassimp-native/src/jassimp.h index f448dc2c2..7d4b66e29 100644 --- a/port/jassimp/jassimp-native/src/jassimp.h +++ b/port/jassimp/jassimp-native/src/jassimp.h @@ -39,7 +39,7 @@ JNIEXPORT jstring JNICALL Java_jassimp_Jassimp_getErrorString * Signature: (Ljava/lang/String;J)Ljassimp/AiScene; */ JNIEXPORT jobject JNICALL Java_jassimp_Jassimp_aiImportFile - (JNIEnv *, jclass, jstring, jlong); + (JNIEnv *, jclass, jstring, jlong, jobject); #ifdef __cplusplus } diff --git a/port/jassimp/jassimp/src/jassimp/AiClassLoaderIOSystem.java b/port/jassimp/jassimp/src/jassimp/AiClassLoaderIOSystem.java new file mode 100644 index 000000000..04d638610 --- /dev/null +++ b/port/jassimp/jassimp/src/jassimp/AiClassLoaderIOSystem.java @@ -0,0 +1,130 @@ +/* + * Copyright 2017 Florida Institute for Human and Machine Cognition (IHMC) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jassimp; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; + +/** + * IOSystem based on the Java classloader.

+ * + * This IOSystem allows loading models directly from the + * classpath. No extraction to the file system is + * necessary. + * + * @author Jesper Smith + * + */ +public class AiClassLoaderIOSystem implements AiIOSystem +{ + private final Class clazz; + private final ClassLoader classLoader; + + /** + * Construct a new AiClassLoaderIOSystem.

+ * + * This constructor uses a ClassLoader to resolve + * resources. + * + * @param classLoader classLoader to resolve resources. + */ + public AiClassLoaderIOSystem(ClassLoader classLoader) { + this.clazz = null; + this.classLoader = classLoader; + } + + /** + * Construct a new AiClassLoaderIOSystem.

+ * + * This constructor uses a Class to resolve + * resources. + * + * @param class class to resolve resources. + */ + public AiClassLoaderIOSystem(Class clazz) { + this.clazz = clazz; + this.classLoader = null; + } + + + @Override + public AiInputStreamIOStream open(String filename, String ioMode) { + try { + + InputStream is; + + if(clazz != null) { + is = clazz.getResourceAsStream(filename); + } + else if (classLoader != null) { + is = classLoader.getResourceAsStream(filename); + } + else { + System.err.println("[" + getClass().getSimpleName() + + "] No class or classLoader provided to resolve " + filename); + return null; + } + + if(is != null) { + return new AiInputStreamIOStream(is); + } + else { + System.err.println("[" + getClass().getSimpleName() + + "] Cannot find " + filename); + return null; + } + } + catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public void close(AiInputStreamIOStream file) { + } + + @Override + public boolean exists(String path) + { + URL url = null; + if(clazz != null) { + url = clazz.getResource(path); + } + else if (classLoader != null) { + url = classLoader.getResource(path); + } + + + if(url == null) + { + return false; + } + else + { + return true; + } + + } + + @Override + public char getOsSeparator() + { + return '/'; + } + +} diff --git a/port/jassimp/jassimp/src/jassimp/AiIOStream.java b/port/jassimp/jassimp/src/jassimp/AiIOStream.java new file mode 100644 index 000000000..5378da5f8 --- /dev/null +++ b/port/jassimp/jassimp/src/jassimp/AiIOStream.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017 Florida Institute for Human and Machine Cognition (IHMC) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jassimp; + +import java.nio.ByteBuffer; + + +/** + * Interface to allow custom resource loaders for jassimp.

+ * + * The design is based on passing the file wholly in memory, + * because Java inputstreams do not have to support seek.

+ * + * Writing files from Java is unsupported. + * + * + * @author Jesper Smith + * + */ +public interface AiIOStream +{ + + /** + * Read all data into buffer.

+ * + * The whole stream should be read into the buffer. + * No support is provided for partial reads. + * + * @param buffer Target buffer for the model data + * + * @return true if successful, false if an error occurred. + */ + boolean read(ByteBuffer buffer); + + /** + * The total size of this stream.

+ * + * @return total size of this stream + */ + int getFileSize(); + +} diff --git a/port/jassimp/jassimp/src/jassimp/AiIOSystem.java b/port/jassimp/jassimp/src/jassimp/AiIOSystem.java new file mode 100644 index 000000000..d2c741529 --- /dev/null +++ b/port/jassimp/jassimp/src/jassimp/AiIOSystem.java @@ -0,0 +1,54 @@ +/* + * Copyright 2017 Florida Institute for Human and Machine Cognition (IHMC) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jassimp; + +public interface AiIOSystem +{ + /** + * + * Open a new file with a given path. + * When the access to the file is finished, call close() to release all associated resources + * + * @param path Path to the file + * @param ioMode file I/O mode. Required are: "wb", "w", "wt", "rb", "r", "rt". + * + * @return AiIOStream or null if an error occurred + */ + public T open(String path, String ioMode); + + + /** + * Tests for the existence of a file at the given path. + * + * @param path path to the file + * @return true if there is a file with this path, else false. + */ + public boolean exists(String path); + + /** + * Returns the system specific directory separator.

+ * + * @return System specific directory separator + */ + public char getOsSeparator(); + + /** + * Closes the given file and releases all resources associated with it. + * + * @param file The file instance previously created by Open(). + */ + public void close(T file); +} diff --git a/port/jassimp/jassimp/src/jassimp/AiInputStreamIOStream.java b/port/jassimp/jassimp/src/jassimp/AiInputStreamIOStream.java new file mode 100644 index 000000000..998401b68 --- /dev/null +++ b/port/jassimp/jassimp/src/jassimp/AiInputStreamIOStream.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017 Florida Institute for Human and Machine Cognition (IHMC) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jassimp; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URL; +import java.nio.ByteBuffer; + + +/** + * Implementation of AiIOStream reading from a InputStream + * + * @author Jesper Smith + * + */ +public class AiInputStreamIOStream implements AiIOStream +{ + private final ByteArrayOutputStream os = new ByteArrayOutputStream(); + + + public AiInputStreamIOStream(URI uri) throws IOException { + this(uri.toURL()); + } + + public AiInputStreamIOStream(URL url) throws IOException { + this(url.openStream()); + } + + public AiInputStreamIOStream(InputStream is) throws IOException { + int read; + byte[] data = new byte[1024]; + while((read = is.read(data, 0, data.length)) != -1) { + os.write(data, 0, read); + } + os.flush(); + + is.close(); + } + + @Override + public int getFileSize() { + return os.size(); + } + + @Override + public boolean read(ByteBuffer buffer) { + ByteBufferOutputStream bos = new ByteBufferOutputStream(buffer); + try + { + os.writeTo(bos); + } + catch (IOException e) + { + e.printStackTrace(); + return false; + } + return true; + } + + /** + * Internal helper class to copy the contents of an OutputStream + * into a ByteBuffer. This avoids a copy. + * + */ + private static class ByteBufferOutputStream extends OutputStream { + + private final ByteBuffer buffer; + + public ByteBufferOutputStream(ByteBuffer buffer) { + this.buffer = buffer; + } + + @Override + public void write(int b) throws IOException + { + buffer.put((byte) b); + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + buffer.put(b, off, len); + } + } +} + diff --git a/port/jassimp/jassimp/src/jassimp/Jassimp.java b/port/jassimp/jassimp/src/jassimp/Jassimp.java index 92f4864c7..d1b4aae4e 100644 --- a/port/jassimp/jassimp/src/jassimp/Jassimp.java +++ b/port/jassimp/jassimp/src/jassimp/Jassimp.java @@ -79,6 +79,20 @@ public final class Jassimp { return importFile(filename, EnumSet.noneOf(AiPostProcessSteps.class)); } + /** + * Imports a file via assimp without post processing. + * + * @param filename the file to import + * @param ioSystem ioSystem to load files, or null for default + * @return the loaded scene + * @throws IOException if an error occurs + */ + public static AiScene importFile(String filename, AiIOSystem ioSystem) + throws IOException { + + return importFile(filename, EnumSet.noneOf(AiPostProcessSteps.class), ioSystem); + } + /** * Imports a file via assimp. @@ -89,12 +103,28 @@ public final class Jassimp { * @throws IOException if an error occurs */ public static AiScene importFile(String filename, - Set postProcessing) throws IOException { + Set postProcessing) + throws IOException { + return importFile(filename, postProcessing, null); + } + + /** + * Imports a file via assimp. + * + * @param filename the file to import + * @param postProcessing post processing flags + * @param ioSystem ioSystem to load files, or null for default + * @return the loaded scene, or null if an error occurred + * @throws IOException if an error occurs + */ + public static AiScene importFile(String filename, + Set postProcessing, AiIOSystem ioSystem) + throws IOException { loadLibrary(); return aiImportFile(filename, AiPostProcessSteps.toRawValue( - postProcessing)); + postProcessing), ioSystem); } @@ -310,7 +340,7 @@ public final class Jassimp { * @throws IOException if an error occurs */ private static native AiScene aiImportFile(String filename, - long postProcessing) throws IOException; + long postProcessing, AiIOSystem ioSystem) throws IOException; /** From a7c1dde56e3ae4af5eaef801e938ee9e2265ecfb Mon Sep 17 00:00:00 2001 From: Jesper Smith Date: Fri, 28 Jul 2017 15:08:59 -0500 Subject: [PATCH 298/305] Added return statement to Write --- port/jassimp/jassimp-native/src/jassimp.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/port/jassimp/jassimp-native/src/jassimp.cpp b/port/jassimp/jassimp-native/src/jassimp.cpp index 475f6c5a0..75b1bc510 100644 --- a/port/jassimp/jassimp-native/src/jassimp.cpp +++ b/port/jassimp/jassimp-native/src/jassimp.cpp @@ -471,7 +471,10 @@ public: return cnt; }; - size_t Write(const void* pvBuffer, size_t pSize, size_t pCount) {}; + size_t Write(const void* pvBuffer, size_t pSize, size_t pCount) + { + return 0; + }; aiReturn Seek(size_t pOffset, aiOrigin pOrigin) { From e662f2dc6f5e073a8ae31e9d9b68d4b41267cdee Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 13 Oct 2017 22:41:38 +0200 Subject: [PATCH 299/305] Blender: fix short overflow. --- code/BlenderDNA.inl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/BlenderDNA.inl b/code/BlenderDNA.inl index 6919c9153..8b669180c 100644 --- a/code/BlenderDNA.inl +++ b/code/BlenderDNA.inl @@ -589,7 +589,10 @@ template <> inline void Structure :: Convert (short& dest,const FileData { // automatic rescaling from short to float and vice versa (seems to be used by normals) if (name == "float") { - dest = static_cast(db.reader->GetF4() * 32767.f); + float f = db.reader->GetF4(); + if ( f > 1.0f ) + f = 1.0f; + dest = static_cast( f * 32767.f); //db.reader->IncPtr(-4); return; } From f56e28ea3c55bbc74d0e9a808104bda9b39b2bbc Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 14 Oct 2017 16:40:39 +0200 Subject: [PATCH 300/305] UnitTest: use correct initialized normals in unittest. --- test/unit/utPretransformVertices.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/unit/utPretransformVertices.cpp b/test/unit/utPretransformVertices.cpp index 9f40f3ddc..437cbdba5 100644 --- a/test/unit/utPretransformVertices.cpp +++ b/test/unit/utPretransformVertices.cpp @@ -90,28 +90,36 @@ void PretransformVerticesTest::SetUp() // add 5 empty materials scene->mMaterials = new aiMaterial*[scene->mNumMaterials = 5]; - for (unsigned int i = 0; i < 5;++i) + for (unsigned int i = 0; i < 5;++i) { scene->mMaterials[i] = new aiMaterial(); + } // add 25 test meshes scene->mMeshes = new aiMesh*[scene->mNumMeshes = 25]; - for (unsigned int i = 0; i < 25;++i) { - aiMesh* mesh = scene->mMeshes[i] = new aiMesh(); + for ( unsigned int i = 0; i < 25; ++i) { + aiMesh* mesh = scene->mMeshes[ i ] = new aiMesh(); mesh->mPrimitiveTypes = aiPrimitiveType_POINT; mesh->mFaces = new aiFace[ mesh->mNumFaces = 10+i ]; mesh->mVertices = new aiVector3D[mesh->mNumVertices = mesh->mNumFaces]; for (unsigned int a = 0; a < mesh->mNumFaces; ++a ) { - aiFace& f = mesh->mFaces[a]; - f.mIndices = new unsigned int [f.mNumIndices = 1]; + aiFace& f = mesh->mFaces[ a ]; + f.mIndices = new unsigned int [ f.mNumIndices = 1 ]; f.mIndices[0] = a*3; mesh->mVertices[a] = aiVector3D((float)i,(float)a,0.f); } mesh->mMaterialIndex = i%5; - if (i % 2) + if (i % 2) { mesh->mNormals = new aiVector3D[mesh->mNumVertices]; + for ( unsigned int normalIdx=0; normalIdxmNumVertices; ++normalIdx ) { + mesh->mNormals[ normalIdx ].x = 1.0f; + mesh->mNormals[ normalIdx ].y = 1.0f; + mesh->mNormals[ normalIdx ].z = 1.0f; + mesh->mNormals[ normalIdx ].Normalize(); + } + } } // construct some nodes (1+25) From 6efe4e2841d710a7b3db44a9264fc8c2f52f1990 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 17:13:08 +0300 Subject: [PATCH 301/305] CMake: Add support for Undefined Behavior sanitizer --- CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c30278b7f..bc3731bbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,6 +86,10 @@ OPTION ( ASSIMP_ASAN "Enable AddressSanitizer." OFF ) +OPTION ( ASSIMP_UBSAN + "Enable Undefined Behavior sanitizer." + OFF +) OPTION ( SYSTEM_IRRXML "Use system installed Irrlicht/IrrXML library." OFF @@ -234,6 +238,12 @@ if (ASSIMP_ASAN) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") endif() +if (ASSIMP_UBSAN) + MESSAGE(STATUS "Undefined Behavior sanitizer enabled") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all") +endif() + INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) From 7b73fe8b026ec2dd55fb88a7fe8eccd971f0aad5 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 17:47:06 +0300 Subject: [PATCH 302/305] Travis: Add Clang UBSan build configuration --- .travis.sh | 4 ++++ .travis.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.travis.sh b/.travis.sh index f4ef271e4..9786c5321 100755 --- a/.travis.sh +++ b/.travis.sh @@ -26,6 +26,10 @@ function generate() OPTIONS="$OPTIONS -DASSIMP_ASAN=OFF" fi + if [ "$UBSAN" = "ON" ] ; then + OPTIONS="$OPTIONS -DASSIMP_UBSAN=ON" + fi + cmake -G "Unix Makefiles" $OPTIONS } diff --git a/.travis.yml b/.travis.yml index 3ffa63176..fa37e5955 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,9 @@ matrix: - os: linux compiler: clang env: ASAN=ON + - os: linux + compiler: clang + env: UBSAN=ON - os: linux compiler: clang env: SHARED_BUILD=ON From 7cbb5f4d3b7772861044876d570d142f9fc510cc Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 18:36:09 +0300 Subject: [PATCH 303/305] B3DImporter: Replace bad pointer casting with memcpy --- code/B3DImporter.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/B3DImporter.cpp b/code/B3DImporter.cpp index d15676128..bc888fb66 100644 --- a/code/B3DImporter.cpp +++ b/code/B3DImporter.cpp @@ -171,7 +171,8 @@ int B3DImporter::ReadByte(){ // ------------------------------------------------------------------------------------------------ int B3DImporter::ReadInt(){ if( _pos+4<=_buf.size() ){ - int n=*(int*)&_buf[_pos]; + int n; + memcpy(&n, &_buf[_pos], 4); _pos+=4; return n; } @@ -182,7 +183,8 @@ int B3DImporter::ReadInt(){ // ------------------------------------------------------------------------------------------------ float B3DImporter::ReadFloat(){ if( _pos+4<=_buf.size() ){ - float n=*(float*)&_buf[_pos]; + float n; + memcpy(&n, &_buf[_pos], 4); _pos+=4; return n; } From 9a6b141568c33c01afd9aeac0a283a72dffcf496 Mon Sep 17 00:00:00 2001 From: Turo Lamminen Date: Sat, 7 Oct 2017 20:40:35 +0300 Subject: [PATCH 304/305] FBX: Replace bad pointer casting with memcpy --- code/FBXBinaryTokenizer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/FBXBinaryTokenizer.cpp b/code/FBXBinaryTokenizer.cpp index ede32d7b7..9ae38386b 100644 --- a/code/FBXBinaryTokenizer.cpp +++ b/code/FBXBinaryTokenizer.cpp @@ -151,7 +151,8 @@ uint32_t ReadWord(const char* input, const char*& cursor, const char* end) TokenizeError("cannot ReadWord, out of bounds",input, cursor); } - uint32_t word = *reinterpret_cast(cursor); + uint32_t word; + memcpy(&word, cursor, 4); AI_SWAP4(word); cursor += k_to_read; From f8c40022941178a58deb8161eef493e98fcb6bed Mon Sep 17 00:00:00 2001 From: Jared Mulconry Date: Sat, 14 Oct 2017 22:45:00 +1100 Subject: [PATCH 305/305] Fixed a divide by zero error in IFCBoolean that was latent, but nevertheless a bug --- code/IFCBoolean.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/IFCBoolean.cpp b/code/IFCBoolean.cpp index 8571a3c79..d250fbe36 100644 --- a/code/IFCBoolean.cpp +++ b/code/IFCBoolean.cpp @@ -272,7 +272,6 @@ bool IntersectsBoundaryProfile(const IfcVector3& e0, const IfcVector3& e1, const const IfcVector3& b0 = boundary[i]; const IfcVector3& b1 = boundary[(i + 1) % bcount]; IfcVector3 b = b1 - b0; - IfcFloat b_sqlen_inv = 1.0 / b.SquareLength(); // segment-segment intersection // solve b0 + b*s = e0 + e*t for (s,t) @@ -281,6 +280,7 @@ bool IntersectsBoundaryProfile(const IfcVector3& e0, const IfcVector3& e1, const // no solutions (parallel lines) continue; } + IfcFloat b_sqlen_inv = 1.0 / b.SquareLength(); const IfcFloat x = b0.x - e0.x; const IfcFloat y = b0.y - e0.y;