diff --git a/README.md b/README.md index d7e4f86816..5947384108 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Glaze requires C++20, using concepts for cleaner code and more helpful errors. - Nearly zero intermediate allocations - Powerful wrappers to modify read/write behavior ([Wrappers](./docs/wrappers.md)) - Use your own custom read/write functions ([Custom Read/Write](#custom-readwrite)) +- [Handle unknown keys](./docs/unknown-keys.md) in a fast and flexible manner - Direct memory access through JSON pointer syntax - [Tagged binary spec](./docs/binary.md) through the same API for maximum performance - No exceptions (compiles with `-fno-exceptions`) diff --git a/docs/unknown-keys.md b/docs/unknown-keys.md new file mode 100644 index 0000000000..55fcfb629f --- /dev/null +++ b/docs/unknown-keys.md @@ -0,0 +1,97 @@ +# Unknown Keys + +Sometimes you don't know in advance all the keys of your objects. By default, glaze will error on unknown keys. +If you set the compile time option `error_on_unknown_keys = false`, the behavior will change to skip the unknown keys. + +Furthermore it is possible to customize the handling of object unknown keys at read and/or write time by defining `unknown_read` and `unknown_write` members of meta class as pointer to member or pointer to method. + +Note : `unknown_write` is using `glz::merge` internally so unknown keys will be written at the end of object. + +## Example + +```c++ +struct my_struct +{ + std::string hello; + std::string zzz; + std::map extra; // store key/raw_json in extra map + + // with methods + // void my_unknown_read(const glz::sv& key, const glz::raw_json& value) { + // extra[key] = value; + // }; + // std::map my_unknown_write() const { + // return extra; + // } + +}; + +template <> +struct glz::meta { + using T = my_struct; + static constexpr auto value = object( + "hello", &T::hello, + "zzz", &T::zzz + ); + + // with members + static constexpr auto unknown_write{&T::extra}; + static constexpr auto unknown_read{&T::extra}; + // with methods + // static constexpr auto unknown_write{&T::my_unknown_write}; + // static constexpr auto unknown_read{&T::my_unknown_read}; + +}; + +// usage +std::string buffer = R"({"hello":"Hello World!","lang":"en","zzz":"zzz"})"; +my_struct s{}; + +// decodes and retains extra unknown fields (lang) in extra map +glz::context ctx{}; +glz::read(s, buffer, ctx); + +// update +s.hello = "Hi !" + +// encodes output string +std::string out{}; +glz::write_json(s, out); + +// out == {"hello":"Hi !","zzz":"zzz","lang":"en"} +``` + +## Example 2 : Known extra type + +If the type of extra keys is known, this would work + +```c++ +struct my_struct +{ + std::string infinite; + int zero; + std::map numbers; +}; + +template <> +struct glz::meta { + using T = my_struct; + static constexpr auto value = object( + "infinite", &T::infinite, + "zero", &T::zero + ); + + static constexpr auto unknown_read{&T::extra}; +}; + +// usage +// note extra keys (one, two) are of type int +std::string buffer = R"({"inf":"infinite","zero":0,"one":1,"two":2})"; +my_struct s{}; + +// decodes and retains extra unknown fields (lang) in extra map +glz::context ctx{}; +glz::read(s, buffer, ctx); +``` + + diff --git a/include/glaze/core/meta.hpp b/include/glaze/core/meta.hpp index 24cfceb674..ceaed038b8 100644 --- a/include/glaze/core/meta.hpp +++ b/include/glaze/core/meta.hpp @@ -68,6 +68,12 @@ namespace glz template concept glaze_t = requires { meta>::value; } || local_meta_t>; + + template + concept has_unknown_writer = requires { meta::unknown_write; } || requires { T::glaze::unknown_write; }; + + template + concept has_unknown_reader = requires { meta::unknown_read; } || requires { T::glaze::unknown_read; }; } struct empty @@ -123,6 +129,38 @@ namespace glz template using remove_meta_wrapper_t = typename remove_meta_wrapper::type; + template + inline constexpr auto meta_unknown_write_v = [] { + if constexpr (detail::local_meta_t) { + return T::glaze::unknown_write; + } + else if constexpr (detail::global_meta_t) { + return meta::unknown_write; + } + else { + return empty{}; + } + }(); + + template + using meta_unknown_write_t = std::decay_t>)>; + + template + inline constexpr auto meta_unknown_read_v = [] { + if constexpr (detail::local_meta_t) { + return T::glaze::unknown_read; + } + else if constexpr (detail::global_meta_t) { + return meta::unknown_read; + } + else { + return empty{}; + } + }(); + + template + using meta_unknown_read_t = std::decay_t>)>; + template concept named = requires { meta::name; } || requires { T::glaze::name; }; diff --git a/include/glaze/core/opts.hpp b/include/glaze/core/opts.hpp index a20ee60828..d24856b02a 100644 --- a/include/glaze/core/opts.hpp +++ b/include/glaze/core/opts.hpp @@ -36,6 +36,7 @@ namespace glz bool closing_handled = false; // the closing character has been handled bool ws_handled = false; // whitespace has already been parsed bool no_header = false; // whether or not a binary header is needed + bool write_unknown = true; // whether to write unkwown fields }; template @@ -109,4 +110,20 @@ namespace glz template inline constexpr auto opt_false = opt_off(); + + template + constexpr auto write_unknown_off() + { + opts ret = Opts; + ret.write_unknown = false; + return ret; + } + + template + constexpr auto write_unknown_on() + { + opts ret = Opts; + ret.write_unknown = true; + return ret; + } } diff --git a/include/glaze/json/read.hpp b/include/glaze/json/read.hpp index 782b46dd59..78851e0528 100644 --- a/include/glaze/json/read.hpp +++ b/include/glaze/json/read.hpp @@ -78,8 +78,61 @@ namespace glz } } } + + template + GLZ_ALWAYS_INLINE static void handle_unknown(const glz::sv& key, T&& value, Ctx&& ctx, It0&& it, It1&& end) noexcept + { + using ValueType = std::decay_t; + if constexpr (detail::has_unknown_reader) { + constexpr auto& reader = meta_unknown_read_v; + using ReaderType = meta_unknown_read_t; + if constexpr (std::is_member_object_pointer_v) + { + using MemberType = typename member_value::type; + if constexpr (detail::map_subscriptable) + { + read::op((value.*reader)[key], ctx, it, end); + } + else + { + static_assert(false_v, "target must have subscript operator"); + } + } + else if constexpr (std::is_member_function_pointer_v) + { + using ReturnType = typename return_type::type; + if constexpr (std::is_void_v) { + using TupleType = typename inputs_as_tuple::type; + if constexpr (std::tuple_size_v == 2) { + std::decay_t> input{}; + read::op(input, ctx, it, end); + if (bool(ctx.error)) [[unlikely]] + return; + (value.*reader)(key, input); + } + else + { + static_assert(false_v, "method must have 2 args"); + } + } + else + { + static_assert(false_v, "method must have void return"); + } + } + else + { + static_assert(false_v, "unknown_read type not handled"); + } + } + else + { + skip_value(ctx, it, end); + } + } }; + template struct from_json { @@ -1572,7 +1625,7 @@ namespace glz if (bool(ctx.error)) [[unlikely]] return; - skip_value(ctx, it, end); + read::handle_unknown(key, value, ctx, it, end); if (bool(ctx.error)) [[unlikely]] return; } @@ -1624,7 +1677,7 @@ namespace glz if (bool(ctx.error)) [[unlikely]] return; - skip_value(ctx, it, end); + read::handle_unknown(key, value, ctx, it, end); if (bool(ctx.error)) [[unlikely]] return; } @@ -1635,7 +1688,7 @@ namespace glz if (bool(ctx.error)) [[unlikely]] return; - skip_value(ctx, it, end); + read::handle_unknown(key, value, ctx, it, end); if (bool(ctx.error)) [[unlikely]] return; } diff --git a/include/glaze/json/write.hpp b/include/glaze/json/write.hpp index 9ec084f070..d909f5dcf0 100644 --- a/include/glaze/json/write.hpp +++ b/include/glaze/json/write.hpp @@ -933,8 +933,36 @@ namespace glz requires glaze_object_t struct to_json { + template + GLZ_FLATTEN static void op(V&& value, is_context auto&& ctx, auto&& b, auto&& ix) noexcept + { + using ValueType = std::decay_t; + if constexpr (detail::has_unknown_writer && Options.write_unknown) { + constexpr auto& writer = meta_unknown_write_v; + + using WriterType = meta_unknown_write_t; + if constexpr (std::is_member_object_pointer_v) + { + write::op()>(glz::merge{value, value.*writer}, ctx, b, ix); + } + else if constexpr (std::is_member_function_pointer_v) + { + write::op()>(glz::merge{value, (value.*writer)()}, ctx, b, ix); + } + else + { + static_assert(false_v, "unknown_write type not handled"); + } + } + else + { + op_base()>(std::forward(value), ctx, b, ix); + } + } + + // handles glaze_object_t without extra unknown fields template - GLZ_FLATTEN static void op(auto&& value, is_context auto&& ctx, auto&& b, auto&& ix) noexcept + GLZ_FLATTEN static void op_base(auto&& value, is_context auto&& ctx, auto&& b, auto&& ix) noexcept { if constexpr (!Options.opening_handled) { dump<'{'>(b, ix); diff --git a/tests/json_test/json_test.cpp b/tests/json_test/json_test.cpp index a2163cfe64..0e70d2416a 100644 --- a/tests/json_test/json_test.cpp +++ b/tests/json_test/json_test.cpp @@ -5949,6 +5949,154 @@ suite float128_test = [] { }; #endif +struct unknown_fields_member +{ + std::string a; + std::string missing; + std::string end; + std::map extra; +}; + +template <> +struct glz::meta +{ + using T = unknown_fields_member; + static constexpr auto value = object( + "a", &T::a, + "missing", &T::missing, + "end", &T::end + ); + static constexpr auto unknown_write{&T::extra}; + static constexpr auto unknown_read{&T::extra}; +}; + +suite unknown_fields_member_test = [] { + "decode_unknown"_test = [] { + unknown_fields_member obj{}; + + std::string buffer = R"({"a":"aaa","unk":"zzz", "unk2":{"sub":3,"sub2":[{"a":"b"}]},"unk3":[], "end":"end"})"; + + glz::context ctx{}; + + expect(!glz::read(obj, buffer, ctx)); + + expect(obj.extra["unk"].str == R"("zzz")"); + expect(obj.extra["unk2"].str == R"({"sub":3,"sub2":[{"a":"b"}]})"); + expect(obj.extra["unk3"].str == R"([])"); + + }; + + "encode_unknown"_test = [] { + unknown_fields_member obj{}; + glz::context ctx{}; + obj.a = "aaa"; + obj.end = "end"; + obj.extra["unk"] = R"("zzz")"; + obj.extra["unk2"] = R"({"sub":3,"sub2":[{"a":"b"}]})"; + obj.extra["unk3"] = R"([])"; + + std::string result = R"({"a":"aaa","missing":"","end":"end","unk":"zzz","unk2":{"sub":3,"sub2":[{"a":"b"}]},"unk3":[]})"; + expect(glz::write_json(obj) == result); + }; +}; + +struct unknown_fields_method +{ + std::string a; + std::string missing; + std::string end; + unknown_fields_member sub; // test writing of sub extras too + std::map extra; + + void my_unknown_read(const glz::sv& key, const glz::raw_json& value) { + extra[key] = value; + }; + + std::map my_unknown_write() const { + return extra; + } + +}; + +template <> +struct glz::meta +{ + using T = unknown_fields_method; + static constexpr auto value = object( + "a", &T::a, + "missing", &T::missing, + "end", &T::end, + "sub", &T::sub + ); + static constexpr auto unknown_write{&T::my_unknown_write}; + static constexpr auto unknown_read{&T::my_unknown_read}; +}; + +suite unknown_fields_method_test = [] { + "decode_unknown"_test = [] { + unknown_fields_method obj{}; + + std::string buffer = R"({"a":"aaa","unk":"zzz", "unk2":{"sub":3,"sub2":[{"a":"b"}]},"unk3":[], "end":"end"})"; + + glz::context ctx{}; + + expect(!glz::read(obj, buffer, ctx)); + + expect(obj.extra["unk"].str == R"("zzz")"); + expect(obj.extra["unk2"].str == R"({"sub":3,"sub2":[{"a":"b"}]})"); + expect(obj.extra["unk3"].str == R"([])"); + + }; + + "encode_unknown"_test = [] { + unknown_fields_method obj{}; + glz::context ctx{}; + obj.a = "aaa"; + obj.end = "end"; + obj.my_unknown_read("unk", R"("zzz")"); + obj.my_unknown_read("unk2", R"({"sub":3,"sub2":[{"a":"b"}]})"); + obj.my_unknown_read("unk3", R"([])"); + obj.sub.extra["subextra"] = R"("subextraval")"; + std::string result = R"({"a":"aaa","missing":"","end":"end","sub":{"a":"","missing":"","end":"","subextra":"subextraval"},"unk":"zzz","unk2":{"sub":3,"sub2":[{"a":"b"}]},"unk3":[]})"; + expect(glz::write_json(obj) == result); + }; +}; + +struct unknown_fields_known_type +{ + std::string a; + std::string missing; + std::string end; + std::map extra; +}; + +template <> +struct glz::meta +{ + using T = unknown_fields_known_type; + static constexpr auto value = object( + "a", &T::a, + "missing", &T::missing, + "end", &T::end + ); + static constexpr auto unknown_write{&T::extra}; + static constexpr auto unknown_read{&T::extra}; +}; + +suite unknown_fields_known_type_test = [] { + "decode_unknown"_test = [] { + std::string buffer = R"({"a":"aaa","unk":5, "unk2":22,"unk3":355, "end":"end"})"; + + unknown_fields_known_type obj{}; + expect(!glz::read(obj, buffer)); + + expect(obj.extra["unk"] == 5); + expect(obj.extra["unk2"] == 22); + expect(obj.extra["unk3"] == 355); + }; +}; + + int main() { // Explicitly run registered test suites and report errors