From 0fb7b0986dff09671abe55dd11acbc950573e8d3 Mon Sep 17 00:00:00 2001 From: fraillt Date: Tue, 7 Feb 2017 07:26:23 +0200 Subject: [PATCH] initial concept commit --- .gitignore | 3 + CMakeLists.txt | 11 ++ README.md | 4 + ext/gtest/CMakeLists.txt | 57 ++++++++++ include/BufferReader.h | 111 +++++++++++++++++++ include/BufferWriter.h | 137 +++++++++++++++++++++++ include/Common.h | 106 ++++++++++++++++++ include/DeltaDeserializer.h | 185 +++++++++++++++++++++++++++++++ include/DeltaSerializer.h | 182 ++++++++++++++++++++++++++++++ include/Deserializer.h | 116 +++++++++++++++++++ include/Serializer.h | 120 ++++++++++++++++++++ tests/BufferBitsOpTests.cpp | 107 ++++++++++++++++++ tests/BufferBytesOpTests.cpp | 129 +++++++++++++++++++++ tests/CMakeLists.txt | 50 +++++++++ tests/SerializerObjectsTests.cpp | 160 ++++++++++++++++++++++++++ tests/SerializerValuesTests.cpp | 55 +++++++++ 16 files changed, 1533 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 ext/gtest/CMakeLists.txt create mode 100644 include/BufferReader.h create mode 100644 include/BufferWriter.h create mode 100644 include/Common.h create mode 100644 include/DeltaDeserializer.h create mode 100644 include/DeltaSerializer.h create mode 100644 include/Deserializer.h create mode 100644 include/Serializer.h create mode 100644 tests/BufferBitsOpTests.cpp create mode 100644 tests/BufferBytesOpTests.cpp create mode 100644 tests/CMakeLists.txt create mode 100644 tests/SerializerObjectsTests.cpp create mode 100644 tests/SerializerValuesTests.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8a579ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea/ +build/ +cmake-build-debug/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..8d1ad7e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.2) + +project(serialization) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +#add tests +enable_testing() +add_subdirectory(tests) diff --git a/README.md b/README.md new file mode 100644 index 0000000..e8995c0 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# Bitsery # +Header only C++ byte and bit serialization library. + +Currently it is not usable diff --git a/ext/gtest/CMakeLists.txt b/ext/gtest/CMakeLists.txt new file mode 100644 index 0000000..d54109a --- /dev/null +++ b/ext/gtest/CMakeLists.txt @@ -0,0 +1,57 @@ +cmake_minimum_required(VERSION 3.2) +project(gtest_builder C CXX) +include(ExternalProject) + +set(GTEST_FORCE_SHARED_CRT ON) +set(GTEST_DISABLE_PTHREADS OFF) + +if(MINGW) + set(GTEST_DISABLE_PTHREADS ON) +endif() + +if (${USE_GMOCK}) + message("use gmock") + set(BUILD_ARGS -DBUILD_GTEST=OFF -DBUILD_GMOCK=ON) +else () + message("use gtest only") + set(BUILD_ARGS -DBUILD_GTEST=ON -DBUILD_GMOCK=OFF) +endif() + +if (WIN32 AND NOT MINGW) + set(BUILD_ARGS ${BUILD_ARGS} + -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG:PATH=DebugLibs + -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE:PATH=ReleaseLibs) +endif() + +ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest.git + CMAKE_ARGS ${BUILD_ARGS} + -Dgtest_force_shared_crt=${GTEST_FORCE_SHARED_CRT} + -Dgtest_disable_pthreads=${GTEST_DISABLE_PTHREADS} + PREFIX "${CMAKE_CURRENT_BINARY_DIR}" + # disable update command + UPDATE_COMMAND "" + # disable install step + INSTALL_COMMAND "" + ) + +#export variables +ExternalProject_Get_Property(googletest source_dir) +ExternalProject_Get_Property(googletest binary_dir) +if (${USE_GMOCK}) + # need to include both googletest and googlemock + set(GTEST_INCLUDE_DIRS ${source_dir}/googlemock/include ${source_dir}/googletest/include PARENT_SCOPE) + set(GTEST_LIBS_DIR ${binary_dir}/googlemock PARENT_SCOPE) + set(GTEST_LIBNAME gmock PARENT_SCOPE) + set(GTEST_MAIN_LIBNAME gmock_main PARENT_SCOPE) + set(GTEST_LINK_LIBNAMES gmock_main PARENT_SCOPE) +else() + set(GTEST_INCLUDE_DIRS ${source_dir}/googletest/include PARENT_SCOPE) + set(GTEST_LIBS_DIR ${binary_dir}/googletest PARENT_SCOPE) + set(GTEST_LIBNAME gtest PARENT_SCOPE) + set(GTEST_MAIN_LIBNAME gtest_main PARENT_SCOPE) + # need to include both libs gtest and gtest_main + set(GTEST_LINK_LIBNAMES gtest gtest_main PARENT_SCOPE) +endif() + + diff --git a/include/BufferReader.h b/include/BufferReader.h new file mode 100644 index 0000000..f4ac638 --- /dev/null +++ b/include/BufferReader.h @@ -0,0 +1,111 @@ +// +// Created by Mindaugas Vinkelis on 2016-11-11. +// + +#ifndef PROJECT_TEMPLATE_BUFFER_READER_H +#define PROJECT_TEMPLATE_BUFFER_READER_H + +#include "Common.h" + +#include + +struct BufferReader { + + using value_type = uint8_t; + BufferReader(const std::vector& buf):_buf{buf}, _pos{std::begin(buf)}{ + + } + + template + bool ReadBytes(T& v) { + static_assert(std::is_integral(), ""); + static_assert(sizeof(T) == SIZE, ""); + using UT = typename std::make_unsigned::type; + return m_scratch + ? ReadBits(reinterpret_cast(v)) + : directRead(&v, 1); + } + + template + bool ReadBuffer(T* buf, size_t count) { + static_assert(std::is_integral(), ""); + static_assert(sizeof(T) == SIZE, ""); + + if (m_scratchBits) { + //todo implement +// using UT = typename std::make_unsigned::type; +// WriteBits(reinterpret_cast(v)); + } else { + return directRead(buf, count); + } + return true; + } + + + template + bool ReadBits(T& v) { + static_assert(std::is_integral() && std::is_unsigned(), ""); + static_assert(SIZE > 0 && SIZE <= BITS_SIZE, ""); + + const auto bytesRequired = SIZE > m_scratchBits + ? ((SIZE - 1 - m_scratchBits) >> 3) + 1u + : 0u; + if (static_cast(std::distance(_pos, std::end(_buf))) < bytesRequired ) + return false; + ReadBitsInternal(v, SIZE); + return true; + } + + template + void ReadBitsInternal(T& v, size_t size) { + auto bitsLeft = size; + T res{}; + while (bitsLeft > 0) { + auto bits = std::min(bitsLeft, BITS_SIZE); + if ( m_scratchBits < bits ) { + value_type tmp; + + std::copy_n(_pos, 1, reinterpret_cast(&tmp)); + std::advance(_pos, 1); + + m_scratch |= static_cast(tmp) << m_scratchBits; + m_scratchBits += BITS_SIZE; + } + auto shiftedRes = static_cast(m_scratch & ( (static_cast(1)<>= bits; + m_scratchBits -= bits; + bitsLeft -= bits; + } + v = res; + } + + bool isCompleted() const { + return _pos == std::end(_buf); + } + +private: + const std::vector& _buf; + decltype(std::begin(_buf)) _pos; + template + bool directRead(T* v, size_t count) { + static_assert(!std::is_const::value, ""); + const auto bytesCount = sizeof(T) * count; + if (static_cast(std::distance(_pos, std::end(_buf))) < bytesCount) + return false; + std::copy_n(_pos, bytesCount, reinterpret_cast(v)); + std::advance(_pos, bytesCount); + return true; + } + + + using SCRATCH_TYPE = typename BIGGER_TYPE::type; + + SCRATCH_TYPE m_scratch{}; + size_t m_scratchBits{}; ///< Number of bits currently in the scratch buffer. If the user wants to read more bits than this, we have to go fetch another dword from memory. + +}; + + + +#endif //PROJECT_TEMPLATE_BUFFER_READER_H diff --git a/include/BufferWriter.h b/include/BufferWriter.h new file mode 100644 index 0000000..d2962ad --- /dev/null +++ b/include/BufferWriter.h @@ -0,0 +1,137 @@ +// +// Created by Mindaugas Vinkelis on 2016-11-11. +// + +#ifndef PROJECT_TEMPLATE_BUFFER_WRITER_H +#define PROJECT_TEMPLATE_BUFFER_WRITER_H + +#include "Common.h" +#include + +struct MeasureSize { + + template + void WriteBytes(const T& ) { + static_assert(std::is_integral(), ""); + static_assert(sizeof(T) == SIZE, ""); + _bytesCount += SIZE; + } + + template + void WriteBits(const T& ) { + static_assert(std::is_integral() && std::is_unsigned(), ""); + static_assert(SIZE > 0 && SIZE <= BITS_SIZE, ""); + _bytesCount += SIZE * 8; + } + + template + void WriteBuffer(const T* , size_t count) { + static_assert(std::is_integral(), ""); + static_assert(sizeof(T) == SIZE, ""); + _bytesCount += SIZE * count; + } + + size_t getSize() const { + return _bytesCount; + } +private: + size_t _bytesCount{}; + +}; + + +struct BufferWriter { + using value_type = uint8_t; + BufferWriter(std::vector& buffer):_buf{buffer}, _outIt{std::back_inserter(buffer)} { + + } + + template + void WriteBytes(const T& v) { + static_assert(std::is_integral(), ""); + static_assert(sizeof(T) == SIZE, ""); + + if (m_scratchBits) { + using UT = typename std::make_unsigned::type; + WriteBits(reinterpret_cast(v)); + } else { + directWrite(&v,1); + } + } + + template + void WriteBuffer(const T* buf, size_t count) { + static_assert(std::is_integral(), ""); + static_assert(sizeof(T) == SIZE, ""); + if (m_scratchBits) { + //todo implement +// using UT = typename std::make_unsigned::type; +// WriteBits(reinterpret_cast(v)); + } else { + directWrite(buf, count); + } + + } + + template + void WriteBits(const T& v) { + static_assert(std::is_integral() && std::is_unsigned(), ""); + static_assert(SIZE > 0 && SIZE <= BITS_SIZE, ""); + WriteBitsInternal(v, SIZE); + } + + template + void WriteBitsInternal(const T& v, size_t size) { + auto value = v; + auto bitsLeft = size; + while (bitsLeft > 0) { + auto bits = std::min(bitsLeft, BITS_SIZE); + m_scratch |= static_cast( value ) << m_scratchBits; + m_scratchBits += bits; + if ( m_scratchBits >= BITS_SIZE ) { + auto tmp = static_cast(m_scratch & bufTypeMask); + directWrite(&tmp, 1); + m_scratch >>= BITS_SIZE; + m_scratchBits -= BITS_SIZE; + + value >>= BITS_SIZE; + } + bitsLeft -= bits; + } + } + + void Flush() { + if ( m_scratchBits ) + { + auto tmp = static_cast( m_scratch & bufTypeMask ); + directWrite(&tmp, 1); + m_scratch >>= m_scratchBits; + m_scratchBits -= m_scratchBits; + } + } + + +private: + + template + void directWrite(const T* v, size_t count) { + const auto bytesSize = sizeof(T) * count; + const auto pos = _buf.size(); + _buf.resize(pos + bytesSize); + std::copy_n(reinterpret_cast(v), bytesSize, _buf.data()+pos); + } + + const value_type bufTypeMask = 0xFF; + using SCRATCH_TYPE = typename BIGGER_TYPE::type; + std::vector& _buf; + std::back_insert_iterator> _outIt; + SCRATCH_TYPE m_scratch{}; + size_t m_scratchBits{}; + + + //size_t _bufSize{}; + +}; + + +#endif //PROJECT_TEMPLATE_BUFFER_WRITER_H diff --git a/include/Common.h b/include/Common.h new file mode 100644 index 0000000..a3f8ee7 --- /dev/null +++ b/include/Common.h @@ -0,0 +1,106 @@ +// +// Created by fraillt on 17.1.5. +// + +#ifndef TMP_COMMON_H +#define TMP_COMMON_H + +#include + +template +constexpr size_t BITS_SIZE = sizeof(T) << 3; + +template +struct BIGGER_TYPE {}; + +template <> +struct BIGGER_TYPE { + typedef uint16_t type; +}; + +template <> +struct BIGGER_TYPE { + typedef uint32_t type; +}; + +template <> +struct BIGGER_TYPE { + typedef uint64_t type; +}; + +template <> +struct BIGGER_TYPE { + typedef int16_t type; +}; + +template <> +struct BIGGER_TYPE { + typedef int32_t type; +}; + +template <> +struct BIGGER_TYPE { + typedef int64_t type; +}; + +template <> +struct BIGGER_TYPE { + typedef int16_t type; +}; + +template +constexpr size_t DEFAULT_OR_SIZE =SIZE == 0 ? sizeof(T) : SIZE; + +template +struct ProcessAnyType { + template + static void serialize(S& s, T&& v) { + s.value(std::forward(v)); + } +}; + +template <> +struct ProcessAnyType<0> { + template + static void serialize(S& s, T&& v) { + s.object(std::forward(v)); + } +}; + + +#define SERIALIZE(ObjectType) \ +template ::value || std::is_same::value>::type* = nullptr> \ +S& serialize(S& s, T& o) + + +class ObjectMemoryPosition { +public: + + template + ObjectMemoryPosition(const T& oldObj, const T& newObj) + :ObjectMemoryPosition{reinterpret_cast(&oldObj), reinterpret_cast(&newObj), sizeof(T)} + { + } + + template + bool isFieldsEquals(const T& newObjField) { + return *getOldObjectField(newObjField) == newObjField; + } + template + const T* getOldObjectField(const T& field) { + auto offset = reinterpret_cast(&field) - newObj; + return reinterpret_cast(oldObj + offset); + } +private: + + ObjectMemoryPosition(const char* objOld, const char* objNew, size_t ) + :oldObj{objOld}, + newObj{objNew} + { + } + + const char* oldObj; + const char* newObj; +}; + +#endif //TMP_COMMON_H diff --git a/include/DeltaDeserializer.h b/include/DeltaDeserializer.h new file mode 100644 index 0000000..f70953a --- /dev/null +++ b/include/DeltaDeserializer.h @@ -0,0 +1,185 @@ +// +// Created by fraillt on 17.1.10. +// + +#ifndef TMP_DELTADESERIALIZER_H +#define TMP_DELTADESERIALIZER_H + +#include +#include +#include +#include "Common.h" + +template +class DeltaDeserializer { +public: + DeltaDeserializer(Reader& w, const TObj& oldObj, const TObj& newObj) + :_reader{w}, + _oldObj{oldObj}, + _newObj{newObj}, + _objMemPos(std::deque(1, ObjectMemoryPosition{oldObj, newObj})), + _isNewElement{false} + { + }; + + template::value>::type* = nullptr> + DeltaDeserializer& value(T& v) { + if (getChangedState(v)) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + _reader.template ReadBytes(v); + } + return *this; + } + + template + DeltaDeserializer& object(T&& obj) { + if (getChangedState(obj)) + return serialize(*this, std::forward(obj)); + return *this; + } + + template + DeltaDeserializer& text(T&& str) { + return withContainer(str, [this](auto& v) { value<1>(v);}); + } + + + + template + DeltaDeserializer & withArray(std::array &arr, Fnc&& fnc) { + if (getChangedState(arr)) { + if (!_isNewElement) { + const auto old = *_objMemPos.top().getOldObjectField(arr); + processContainer(std::begin(old), std::end(old), std::begin(arr), std::end(arr), fnc); + } else { + for (auto& v:arr) + fnc(v); + } + } + return *this; + } + + + template + DeltaDeserializer& withArray(T (&arr)[N], Fnc&& fnc) { + if (getChangedState(arr)) { + if (!_isNewElement) { + const auto old = *_objMemPos.top().getOldObjectField(arr); + T* tmp = arr; + processContainer(old, old + N, tmp, tmp + N, fnc); + } else { + T* tmp = arr; + for (auto i=0u; i < N; ++i, ++tmp) + fnc(*tmp); + } + } + return *this; + } + + template + DeltaDeserializer& withContainer(T& obj, Fnc&& fnc) { + if(getChangedState(obj)) { + size_t newSize{}; + _reader.template ReadBits<32>(newSize); + if (!_isNewElement) { + auto old = *_objMemPos.top().getOldObjectField(obj); + if (old.size() != newSize) + obj.resize(newSize); + processContainer(std::begin(old), std::end(old), std::begin(obj), std::end(obj), std::forward(fnc)); + } else { + obj.resize(newSize); + for (auto& v:obj) + fnc(v); + } + } + return *this; + } + +private: + Reader& _reader; + + const TObj& _oldObj; + const TObj& _newObj; + std::stack _objMemPos; + bool _isNewElement; + + template + bool getChangedState(T& obj) { + if (!_isNewElement) { + if (!readChangedState()) { + obj = *_objMemPos.top().getOldObjectField(obj); + return false; + } + } + return true; + } + + template + bool getChangedState(T (&arr)[N]) { + if (!_isNewElement) { + if (!readChangedState()) { + auto old = *_objMemPos.top().getOldObjectField(arr); + auto end = arr + N; + auto pOld = old; + for (auto p=arr; p != end; ++p, ++pOld) + *p = *pOld; + return false; + } + } + return true; + } + + template + bool processContainer(TConstIt oldBegin, TConstIt oldEnd, TIt begin, TIt end, Fnc&& fnc) { + auto offset = readIndexOffset(); + auto p = begin; + auto pOld = oldBegin; + for (; p != end && pOld != oldEnd; ++p, ++pOld) { + if (offset) { + *p = *pOld; + --offset; + } else { + _objMemPos.emplace(ObjectMemoryPosition{*pOld, *p}); + fnc(*p); + _objMemPos.pop(); + offset = readIndexOffset(); + } + } + if (offset != 0 && pOld != oldEnd) + return false; + _isNewElement = true; + for (; p != end; ++p, --offset) + fnc(*p); + _isNewElement = false; + return offset == 0; + + } + + bool readChangedState() { + unsigned char res{}; + _reader.template ReadBits<1>(res); + return res; + } + + size_t readIndexOffset() { + //special case, if items are updated sequentialy + unsigned char tmp{}; + _reader.template ReadBits<1>(tmp); + if (tmp) { + return 0u; + } + else { + size_t res{}; + _reader.template ReadBits<1>(tmp); + if (tmp > 0) + _reader.template ReadBits<4>(res); + else + _reader.template ReadBits<32>(res); + return res; + } + + } + +}; + +#endif //TMP_DELTADESERIALIZER_H diff --git a/include/DeltaSerializer.h b/include/DeltaSerializer.h new file mode 100644 index 0000000..9d8083b --- /dev/null +++ b/include/DeltaSerializer.h @@ -0,0 +1,182 @@ +// +// Created by fraillt on 17.1.10. +// + +#ifndef TMP_DELTASERIALIZER_H +#define TMP_DELTASERIALIZER_H + +#include +#include +#include +#include "Common.h" + +template +class DeltaSerializer { +public: + DeltaSerializer(Writter& w, const TObj& oldObj, const TObj& newObj) + :_writter{w}, + _oldObj{oldObj}, + _newObj{newObj}, + _objMemPos(std::deque(1, ObjectMemoryPosition{oldObj, newObj})), + _isNewElement{false} + { + + }; + + template::value>::type* = nullptr> + DeltaSerializer& value(const T& v) { + if (setChangedState(v)) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + _writter.template WriteBytes(v); + } + return *this; + } + + template + DeltaSerializer& object(T&& obj) { + if (setChangedState(obj)) { + serialize(*this, std::forward(obj)); + } + return *this; + } + + template + DeltaSerializer& text(T&& str) { + return withContainer(str, [this](auto& v) { value<1>(v);}); + } + + template + DeltaSerializer & withArray(const std::array &arr, Fnc&& fnc) { + if (setChangedState(arr)) { + if (!_isNewElement) { + const auto& old = *_objMemPos.top().getOldObjectField(arr); + processContainer(std::begin(old), std::end(old), std::begin(arr), std::end(arr), fnc); + } else { + for (auto& v:arr) + fnc(v); + } + } + return *this; + } + + + template + DeltaSerializer& withArray(const T (&arr)[N], Fnc&& fnc) { + if (setChangedState(arr)) { + if (!_isNewElement) { + auto old = *_objMemPos.top().getOldObjectField(arr); + const T* tmp = arr; + processContainer(old, old + N, tmp, tmp + N, fnc); + } else { + const T* tmp = arr; + for (auto i=0u; i < N; ++i, ++tmp) + fnc(*tmp); + } + } + return *this; + } + + template + DeltaSerializer& withContainer(T&& obj, Fnc&& fnc) { + if(setChangedState(obj)) { + _writter.template WriteBits<32>(obj.size()); + if (!_isNewElement) { + auto old = *_objMemPos.top().getOldObjectField(obj); + processContainer(std::begin(old), std::end(old), std::begin(obj), std::end(obj), std::forward(fnc)); + } else { + for (auto& v:obj) + fnc(v); + } + } + return *this; + } + +private: + Writter& _writter; + const TObj& _oldObj; + const TObj& _newObj; + std::stack _objMemPos; + bool _isNewElement; + + template + bool setChangedState(const T& obj) { + if (!_isNewElement) { + auto res = !_objMemPos.top().isFieldsEquals(obj); + writeChangedState(res); + return res; + } + return true; + } + + template + bool setChangedState(const T (&arr)[N]) { + if (!_isNewElement) { + auto old = *_objMemPos.top().getOldObjectField(arr); + auto end = arr + N; + bool changed{}; + for (auto p=arr, pOld=old; p != end; ++p, ++pOld) { + if (!(*p == *pOld)) { + changed = true; + break; + } + } + writeChangedState(changed); + return changed; + } + return true; + } + + + template + void processContainer(const T oldBegin, const T oldEnd, const T begin, const T end, Fnc&& fnc) { + auto misMatch = std::mismatch(oldBegin, oldEnd, begin, end); + auto lastChanged = begin; + while (misMatch.first != oldEnd && misMatch.second != end) { + writeIndexOffset(std::distance(lastChanged, misMatch.second)); + _objMemPos.emplace(ObjectMemoryPosition{*misMatch.first, *misMatch.second}); + fnc(*misMatch.second); + _objMemPos.pop(); + ++misMatch.first; + ++misMatch.second; + lastChanged = misMatch.second; + misMatch = std::mismatch(misMatch.first, oldEnd, misMatch.second, end); + } + auto p = misMatch.second; + //write items left + writeIndexOffset(std::distance(lastChanged, end)); + //write old elements + for (auto pOld = misMatch.first; p != end && pOld != oldEnd; ++p, ++pOld) { + _objMemPos.emplace(ObjectMemoryPosition{*pOld, *p}); + fnc(*p); + _objMemPos.pop(); + } + + //write new elements + _isNewElement = true; + for (; p != end; ++p) + fnc(*p); + _isNewElement = false; + } + + void writeChangedState(bool state) { + _writter.template WriteBits<1>(state ? 1u : 0u); + } + + void writeIndexOffset(const size_t offset) { + //special case, if items are updated sequentialy + if (offset == 0) { + _writter.template WriteBits<1>(1u); + } else { + _writter.template WriteBits<1>(0u); + auto smallOffset = offset < 16; + _writter.template WriteBits<1>(smallOffset ? 1u : 0u); + if (smallOffset) + _writter.template WriteBits<4>(offset); + else + _writter.template WriteBits<32>(offset); + } + } + +}; + +#endif //TMP_DELTASERIALIZER_H diff --git a/include/Deserializer.h b/include/Deserializer.h new file mode 100644 index 0000000..c0c5963 --- /dev/null +++ b/include/Deserializer.h @@ -0,0 +1,116 @@ +// +// Created by Mindaugas Vinkelis on 17.1.9. +// + +#ifndef TMP_DESERIALIZER_H +#define TMP_DESERIALIZER_H + +#include + +template +class Deserializer { +public: + Deserializer(Reader& r):_reader{r} {}; + + template + Deserializer& object(T&& obj) { + return serialize(*this, std::forward(obj)); + } + + template::value>::type* = nullptr> + Deserializer& value(T& v) { + static_assert(std::numeric_limits::is_iec559, ""); + static_assert(std::numeric_limits::is_iec559, ""); + + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + using CT = std::conditional_t::value, uint32_t, uint64_t>; + static_assert(sizeof(CT) == ValueSize, ""); + _reader.template ReadBytes(reinterpret_cast(v)); + return *this; + } + + template::value>::type* = nullptr> + Deserializer& value(T& v) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + using UT = std::underlying_type_t; + _reader.template ReadBytes(reinterpret_cast(v)); + return *this; + } + + template::value>::type* = nullptr> + Deserializer& value(T& v) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + _reader.template ReadBytes(v); + return *this; + } + + template + Deserializer& text(std::basic_string& str) { + size_t size; + readLength(size); + std::vector buf(size); + _reader.template ReadBuffer(buf.data(), size); + str.assign(buf.data(), size); +// str.resize(size); +// if (size) +// _reader.template ReadBuffer(str.data(), size); + return *this; + } + + template + Deserializer & withArray(std::array &arr, Fnc && fnc) { + for (auto& v: arr) + fnc(v); + return *this; + } + + template + Deserializer& withArray(T (&arr)[N], Fnc&& fnc) { + T* tmp = arr; + for (auto i = 0u; i < N; ++i, ++tmp) + fnc(*tmp); + return *this; + } + + template + Deserializer& withContainer(T&& obj, Fnc&& fnc) { + decltype(obj.size()) size{}; + readLength(size); + obj.resize(size); + for (auto& v:obj) + fnc(v); + return *this; + } + + + + template + Deserializer& withContainer(T&& obj) { + decltype(obj.size()) size{}; + readLength(size); + obj.resize(size); + for (auto& v: obj) + object(v); + return *this; + } + + template + Deserializer& withContainer(T&& obj) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + writeLength(obj.size()); + for (auto& v: obj) + value(v); + return *this; + } + + + +private: + Reader& _reader; + void readLength(size_t& size) { + _reader.template ReadBits<32>(size); + } +}; + + +#endif //TMP_DESERIALIZER_H diff --git a/include/Serializer.h b/include/Serializer.h new file mode 100644 index 0000000..5fb2038 --- /dev/null +++ b/include/Serializer.h @@ -0,0 +1,120 @@ +// +// Created by Mindaugas Vinkelis on 17.1.3. +// + +#ifndef TMP_SERIALIZER_H +#define TMP_SERIALIZER_H + +#include + +template +class Serializer { +public: + Serializer(Writter& w):_writter{w} {}; + + template + Serializer& object(T&& obj) { + return serialize(*this, std::forward(obj)); + } + + template::value>::type* = nullptr> + Serializer& value(const T& v) { + static_assert(std::numeric_limits::is_iec559, ""); + static_assert(std::numeric_limits::is_iec559, ""); + + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + using CT = std::conditional_t::value, uint32_t, uint64_t>; + _writter.template WriteBytes(reinterpret_cast(v)); + return *this; + } + + template::value>::type* = nullptr> + Serializer& value(const T& v) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + _writter.template WriteBytes(reinterpret_cast&>(v)); + return *this; + } + + template::value>::type* = nullptr> + Serializer& value(const T& v) { + constexpr size_t ValueSize = SIZE == 0 ? sizeof(T) : SIZE; + _writter.template WriteBytes(v); + return *this; + } + + template + Serializer& text(const std::basic_string& str) { + procText(str.data()); + return *this; + } + + template + Serializer& text(const T (&str)[N]) { + procText(str); + return *this; + } + + + template + Serializer & withArray(const std::array &arr, Fnc&& fnc) { + for (auto& v: arr) + fnc(v); + return *this; + } + + template + Serializer& withArray(const T (&arr)[N], Fnc&& fnc) { + const T* end = arr + N; + for (const T* tmp = arr; tmp != end; ++tmp) + fnc(*tmp); + return *this; + } + + template + Serializer& withContainer(T&& obj, Fnc&& fnc) { + writeLength(obj.size()); + for (auto& v: obj) + fnc(v); + return *this; + } + + template + Serializer& withContainer(T&& obj) { + writeLength(obj.size()); + procContainer<0>(std::forward(obj)); + return *this; + } + + template ::value && std::is_enum::value>::type> + Serializer& withContainer(T&& obj) { + writeLength(obj.size()); + //procContainer(std::forward(obj)); + procContainer>(std::forward(obj)); + return *this; + } + + +private: + Writter& _writter; + void writeLength(const size_t size) { + _writter.template WriteBits<32>(size); + } + + template + void procContainer(T&& obj) { + for (auto& v: obj) + ProcessAnyType::serialize(*this, v); + }; + + template + void procText(const T* str) { + const auto size = std::char_traits::length(str); + writeLength(size); + if (size) + _writter.template WriteBuffer(str, size); + } + +}; + + +#endif //TMP_SERIALIZER_H diff --git a/tests/BufferBitsOpTests.cpp b/tests/BufferBitsOpTests.cpp new file mode 100644 index 0000000..e467fb2 --- /dev/null +++ b/tests/BufferBitsOpTests.cpp @@ -0,0 +1,107 @@ +// +// Created by Mindaugas Vinkelis on 2017-01-10. +// +#include +#include "BufferWriter.h" +#include "BufferReader.h" + +using testing::Eq; +using testing::ContainerEq; + +#include +#include + + +struct IntegralUnsignedTypes { + uint32_t a; + uint16_t b; + uint8_t c; + uint8_t d; + uint64_t e; +}; + +TEST(BufferBitsOperations, WriteAndReadBits) { + //setup data + IntegralUnsignedTypes data; + data.a = 485454;//bits 19 + data.b = 45978;//bits 16 + data.c = 0;//bits 1 + data.d = 36;//bits 6 + data.e = 479845648946;//bits 39 + + constexpr size_t aBITS = 21; + constexpr size_t bBITS = 16; + constexpr size_t cBITS = 5; + constexpr size_t dBITS = 7; + constexpr size_t eBITS = 40; + + //create and write to buffer + std::vector buf; + BufferWriter bw{buf}; + + bw.WriteBits(data.a); + bw.WriteBits(data.b); + bw.WriteBits(data.c); + bw.WriteBits(data.d); + bw.WriteBits(data.e); + bw.Flush(); + auto bytesCount = ((aBITS + bBITS + cBITS + dBITS + eBITS) / 8) +1 ; + EXPECT_THAT(std::distance(buf.begin(), buf.end()), Eq(bytesCount)); + //read from buffer + BufferReader br{buf}; + IntegralUnsignedTypes res; + + br.ReadBits(res.a); + br.ReadBits(res.b); + br.ReadBits(res.c); + br.ReadBits(res.d); + br.ReadBits(res.e); + + EXPECT_THAT(res.a, Eq(data.a)); + EXPECT_THAT(res.b, Eq(data.b)); + EXPECT_THAT(res.c, Eq(data.c)); + EXPECT_THAT(res.d, Eq(data.d)); + EXPECT_THAT(res.e, Eq(data.e)); + +} + +TEST(BufferBitsOperations, WhenFinishedFlushWriter) { + + std::vector buf; + BufferWriter bw{buf}; + + bw.WriteBits<2>(0xFFu); + EXPECT_THAT(std::distance(buf.begin(), buf.end()), Eq(0)); + bw.Flush(); + EXPECT_THAT(std::distance(buf.begin(), buf.end()), Eq(1)); + +} + +TEST(BufferBitsOperations, BufferSizeIsCountedPerByteNotPerBit) { + //setup data + + //create and write to buffer + std::vector buf; + BufferWriter bw{buf}; + + bw.WriteBits<2>(0xFFu); + bw.Flush(); + EXPECT_THAT(std::distance(buf.begin(), buf.end()), Eq(1)); + + //read from buffer + BufferReader br{buf}; + unsigned tmp; + EXPECT_THAT(br.ReadBits<4>(tmp), Eq(true)); + EXPECT_THAT(br.ReadBits<2>(tmp), Eq(true)); + EXPECT_THAT(br.ReadBits<2>(tmp), Eq(true)); + EXPECT_THAT(br.ReadBits<2>(tmp), Eq(false)); + + //part of next byte + BufferReader br1{buf}; + EXPECT_THAT(br1.ReadBits<2>(tmp), Eq(true)); + EXPECT_THAT(br1.ReadBits<7>(tmp), Eq(false)); + + //bigger than byte + BufferReader br2{buf}; + EXPECT_THAT(br2.ReadBits<9>(tmp), Eq(false)); +} diff --git a/tests/BufferBytesOpTests.cpp b/tests/BufferBytesOpTests.cpp new file mode 100644 index 0000000..fc047cc --- /dev/null +++ b/tests/BufferBytesOpTests.cpp @@ -0,0 +1,129 @@ +// +// Created by Mindaugas Vinkelis on 2016-11-11. +// +#include +#include "BufferWriter.h" +#include "BufferReader.h" + +using testing::Eq; +using testing::ContainerEq; + +#include +#include + +struct IntegralTypes { + int64_t a; + uint32_t b; + int16_t c; + uint8_t d; + int8_t e; + int8_t f[2]; +}; + + +TEST(BufferBytesOperations, WriteAndReadBytes) { + //setup data + IntegralTypes data; + data.a = -4894541654564; + data.b = 94545646; + data.c = -8778; + data.d = 200; + data.e = -98; + data.f[0] = 43; + data.f[1] = -45; + + //create and write to buffer + std::vector buf{}; + BufferWriter bw{buf}; + + bw.WriteBytes<4>(data.b); + bw.WriteBytes<1>(data.f[0]); + bw.WriteBytes<2>(data.c); + bw.WriteBytes<1>(data.d); + bw.WriteBytes<8>(data.a); + bw.WriteBytes<1>(data.e); + bw.WriteBytes<1>(data.f[1]); + EXPECT_THAT(std::distance(buf.begin(), buf.end()), Eq(18)); + //read from buffer + BufferReader br{buf}; + IntegralTypes res{}; + EXPECT_THAT(br.ReadBytes<4>(res.b), Eq(true)); + EXPECT_THAT(br.ReadBytes<1>(res.f[0]), Eq(true)); + EXPECT_THAT(br.ReadBytes<2>(res.c), Eq(true)); + EXPECT_THAT(br.ReadBytes<1>(res.d), Eq(true)); + EXPECT_THAT(br.ReadBytes<8>(res.a), Eq(true)); + EXPECT_THAT(br.ReadBytes<1>(res.e), Eq(true)); + EXPECT_THAT(br.ReadBytes<1>(res.f[1]), Eq(true)); + //assert results + + EXPECT_THAT(data.a, Eq(res.a)); + EXPECT_THAT(data.b, Eq(res.b)); + EXPECT_THAT(data.c, Eq(res.c)); + EXPECT_THAT(data.d, Eq(res.d)); + EXPECT_THAT(data.e, Eq(res.e)); + EXPECT_THAT(data.f, ContainerEq(res.f)); + +} + +TEST(BufferBytesOperations, ReadReturnsFalseIfNotEnoughBufferSize) { + //setup data + uint8_t a = 111; + + //create and write to buffer + std::vector buf{}; + BufferWriter bw{buf}; + + bw.WriteBytes<1>(a); + bw.WriteBytes<1>(a); + bw.WriteBytes<1>(a); + //read from buffer + BufferReader br{buf}; + int16_t b; + int32_t c; + EXPECT_THAT(br.ReadBytes<4>(c), Eq(false)); + EXPECT_THAT(br.ReadBytes<2>(b), Eq(true)); + EXPECT_THAT(br.ReadBytes<2>(b), Eq(false)); + EXPECT_THAT(br.ReadBytes<1>(a), Eq(true)); + EXPECT_THAT(br.ReadBytes<1>(a), Eq(false)); + EXPECT_THAT(br.ReadBytes<2>(b), Eq(false)); + EXPECT_THAT(br.ReadBytes<4>(c), Eq(false)); + +} + + +TEST(BufferBytesOperations, ReadIsCompletedWhenAllBytesAreRead) { + //setup data + IntegralTypes data; + data.b = 94545646; + data.c = -8778; + data.d = 200; + + //create and write to buffer + std::vector buf{}; + BufferWriter bw{buf}; + + bw.WriteBytes<4>(data.b); + bw.WriteBytes<2>(data.c); + bw.WriteBytes<1>(data.d); + //read from buffer + BufferReader br{buf}; + IntegralTypes res; + EXPECT_THAT(br.ReadBytes<4>(res.b), Eq(true)); + EXPECT_THAT(br.ReadBytes<2>(res.c), Eq(true)); + EXPECT_THAT(br.isCompleted(), Eq(false)); + EXPECT_THAT(br.ReadBytes<1>(res.d), Eq(true)); + EXPECT_THAT(br.isCompleted(), Eq(true)); + EXPECT_THAT(br.ReadBytes<1>(res.d), Eq(false)); + EXPECT_THAT(br.isCompleted(), Eq(true)); + + BufferReader br1{buf}; + EXPECT_THAT(br1.ReadBytes<4>(res.b), Eq(true)); + EXPECT_THAT(br1.ReadBytes<2>(res.c), Eq(true)); + EXPECT_THAT(br1.isCompleted(), Eq(false)); + EXPECT_THAT(br1.ReadBytes<2>(res.c), Eq(false)); + EXPECT_THAT(br1.isCompleted(), Eq(false)); + EXPECT_THAT(br1.ReadBytes<1>(res.d), Eq(true)); + EXPECT_THAT(br1.isCompleted(), Eq(true)); + + +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..29f76a2 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.2) +set(PROJECT_TEST_NAME serialization_tests) +project(${PROJECT_TEST_NAME} C CXX) + +find_package(Threads REQUIRED) + +#add googletest external project +#USE_GMOCK enable gmock +#exports variables GTEST_INCLUDE_DIRS, GTEST_LIBS_DIR, GTEST_LIBNAME, GTEST_MAIN_LIBNAME +set(EXT_PROJECTS_DIR ${CMAKE_SOURCE_DIR}/ext) +set(USE_GMOCK ON) +add_subdirectory(${EXT_PROJECTS_DIR}/gtest ${CMAKE_BINARY_DIR}/gtest) + +# set common include folder for module +include_directories(SYSTEM ${GTEST_INCLUDE_DIRS}) +include_directories(${CMAKE_SOURCE_DIR}/include) + +file(GLOB TEST_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +add_executable(${PROJECT_TEST_NAME} ${TEST_SRC_FILES}) +add_dependencies(${PROJECT_TEST_NAME} googletest) + +set_property(TARGET ${PROJECT_TEST_NAME} PROPERTY CXX_STANDARD 14) +set_property(TARGET ${PROJECT_TEST_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + + +if(NOT WIN32 OR MINGW) + FOREACH(LIBNAME ${GTEST_LINK_LIBNAMES}) + target_link_libraries(${PROJECT_TEST_NAME} ${GTEST_LIBS_DIR}/lib${LIBNAME}.a ) + ENDFOREACH() +else() + FOREACH(LIBNAME ${GTEST_LINK_LIBNAMES}) + target_link_libraries(${PROJECT_TEST_NAME} + debug ${GTEST_LIBS_DIR}/DebugLibs/${CMAKE_FIND_LIBRARY_PREFIXES}${LIBNAME}${CMAKE_FIND_LIBRARY_SUFFIXES} + optimized ${GTEST_LIBS_DIR}/ReleaseLibs/${CMAKE_FIND_LIBRARY_PREFIXES}${LIBNAME}${CMAKE_FIND_LIBRARY_SUFFIXES}) + ENDFOREACH() +endif() + +target_link_libraries(${PROJECT_TEST_NAME} ${CMAKE_THREAD_LIBS_INIT}) + +add_test(NAME all_tests COMMAND $) + + + + + + + + + + diff --git a/tests/SerializerObjectsTests.cpp b/tests/SerializerObjectsTests.cpp new file mode 100644 index 0000000..3b4c4d3 --- /dev/null +++ b/tests/SerializerObjectsTests.cpp @@ -0,0 +1,160 @@ +// +// Created by fraillt on 17.1.31. +// + +#include +#include +#include +#include "BufferWriter.h" +#include "Serializer.h" + +#include "DeltaSerializer.h" +#include "DeltaDeserializer.h" + +#include +#include + +using testing::Eq; +using testing::StrEq; +using testing::ContainerEq; + +struct X { + X() {}; + X(int v) :x{ v } {} + std::string s{}; + int x{}; + bool operator ==(const X& r) const { + return r.x == x && r.s == s; + } +}; + + +struct Y { + int y{}; + int carr[3]; + std::array arr; + std::vector vx; + std::string s; +}; +struct Z { X x{}; Y y{}; }; + + +SERIALIZE(Z) +{ + s.object(o.x); + s.object(o.y); + return s; +} + +SERIALIZE(X) +{ + return s.value(o.x) + .text(o.s); +} + +SERIALIZE(Y) +{ + auto writeInt = [&s](auto& v) { s.template value<4>(v); }; + s.text(o.s); + s.template value<4>(o.y); + s.withArray(o.arr, writeInt); + s.withArray(o.carr, writeInt); + s.withContainer(o.vx, [&s](auto& v) { s.object(v); }); + return s; +} + + +TEST(Serializer, GeneralConceptTest) { + //std::string buf; + std::vector buf{}; + BufferWriter bw{ buf }; + Serializer ser(bw); + Y y{}; + y.y = 3423; + y.arr[0] = 111; + y.arr[1] = 222; + y.arr[2] = 333; + y.carr[0] = 123; + y.carr[1] = 456; + y.carr[2] = 789; + y.vx.push_back(X(234)); + y.vx.push_back(X(6245)); + y.vx.push_back(X(613461)); + y.s = "labal diena"; + + Z z{}; + z.y = y; + z.x = X{ 234 }; + + + serialize(ser, y); + serialize(ser, z); + + BufferReader br{ buf }; + Deserializer des(br); + + Y yres{}; + Z zres{}; + serialize(des, yres); + serialize(des, zres); + + EXPECT_THAT(yres.y, Eq(y.y)); + EXPECT_THAT(yres.vx, ContainerEq(y.vx)); + EXPECT_THAT(yres.arr, ContainerEq(y.arr)); + EXPECT_THAT(yres.carr, ContainerEq(y.carr)); + EXPECT_THAT(yres.s, StrEq(y.s)); + + EXPECT_THAT(zres.y.y, Eq(z.y.y)); + EXPECT_THAT(zres.y.vx, ContainerEq(z.y.vx)); + EXPECT_THAT(zres.y.arr, ContainerEq(z.y.arr)); + EXPECT_THAT(zres.y.carr, ContainerEq(z.y.carr)); + EXPECT_THAT(zres.y.s, StrEq(z.y.s)); + EXPECT_THAT(zres.x.s, StrEq(z.x.s)); + EXPECT_THAT(zres.x.x, Eq(z.x.x)); + +} + +TEST(DeltaSerializer, GeneralConceptTest) { + //std::string buf; + Y y{}; + y.y = 3423; + y.arr[0] = 111; + y.arr[1] = 222; + y.arr[2] = 333; + y.carr[0] = 123; + y.carr[1] = 456; + y.carr[2] = 789; + y.vx.push_back(X(234)); + y.vx.push_back(X(6245)); + y.vx.push_back(X(613461)); + y.s = "labal diena"; + y.vx[0].s = "very nice"; + y.vx[1].s = "very nice string, that is a little bit longer that previous"; + + Y yRead = y; + Y yNew = y; + yNew.y = 111111; + yNew.arr[2] = 0xFFFFFFFF; + yNew.carr[1] = 0xFFFFFFFF; + yNew.s = "labas dienaABC"; + yNew.vx[0].s = "very nigga"; + yNew.vx[1].s = "bla"; + yNew.vx.push_back(X{ 3 }); + + std::vector buf; + BufferWriter bw{ buf }; + DeltaSerializer ser(bw, y, yNew); + serialize(ser, yNew); + bw.Flush(); + + BufferReader br{ buf }; + DeltaDeserializer des(br, y, yRead); + serialize(des, yRead); + + EXPECT_THAT(yRead.y, Eq(yNew.y)); + EXPECT_THAT(yRead.vx, ContainerEq(yNew.vx)); + EXPECT_THAT(yRead.arr, ContainerEq(yNew.arr)); + EXPECT_THAT(yRead.carr, ContainerEq(yNew.carr)); + EXPECT_THAT(yRead.s, StrEq(yNew.s)); +} + diff --git a/tests/SerializerValuesTests.cpp b/tests/SerializerValuesTests.cpp new file mode 100644 index 0000000..8d9d25b --- /dev/null +++ b/tests/SerializerValuesTests.cpp @@ -0,0 +1,55 @@ +// +// Created by fraillt on 17.1.5. +// + +#include +#include +#include +#include "BufferWriter.h" +#include "Serializer.h" + + +using testing::Eq; + +template +bool SerializeDeserializeValue(const T& v) { + T res{}; + std::vector buf{}; + + BufferWriter bw{buf}; + Serializer ser(bw); + ser.value(v); + bw.Flush(); + + BufferReader br{buf}; + Deserializer des(br); + des.value(res); + return v == res; +} + +TEST(SerializerValues, IntegerTypes) { + EXPECT_THAT(SerializeDeserializeValue(-449874), Eq(true)); + EXPECT_THAT(SerializeDeserializeValue(34u), Eq(true)); +} + +TEST(SerializerValues, EnumTypes) { + enum E1{ + A1,B1,C1,D1 + }; + EXPECT_THAT(SerializeDeserializeValue(E1::C1), Eq(true)); + enum class E2 { + A2,B2,C2,D2 + }; + EXPECT_THAT(SerializeDeserializeValue(E2::B2), Eq(true)); + enum class E3:short { + A3, B3, C3=4568, D3 + }; + EXPECT_THAT(SerializeDeserializeValue(E3::C3), Eq(true)); +} + + +TEST(SerializerValues, FloatingPointTypes) { + EXPECT_THAT(SerializeDeserializeValue(-484.465), Eq(true)); + EXPECT_THAT(SerializeDeserializeValue(0.00000015f), Eq(true)); +} +