Extra/Unknown fields handling (#529)

README.md

@@ -29,6 +29,7 @@ Glaze requires C++20, using concepts for cleaner code and more helpful errors.
 - Nearly zero intermediate allocations
 - Powerful wrappers to modify read/write behavior ([Wrappers](./docs/wrappers.md))
 - Use your own custom read/write functions ([Custom Read/Write](#custom-readwrite))
+- [Handle unknown keys](./docs/unknown-keys.md) in a fast and flexible manner
 - Direct memory access through JSON pointer syntax
 - [Tagged binary spec](./docs/binary.md) through the same API for maximum performance
 - No exceptions (compiles with `-fno-exceptions`)

docs/unknown-keys.md

@@ -0,0 +1,97 @@
+# Unknown Keys
+
+Sometimes you don't know in advance all the keys of your objects. By default, glaze will error on unknown keys.
+If you set the compile time option `error_on_unknown_keys = false`, the behavior will change to skip the unknown keys.
+
+Furthermore it is possible to customize the handling of object unknown keys at read and/or write time by defining `unknown_read` and `unknown_write` members of meta class as pointer to member or pointer to method.
+
+Note : `unknown_write` is using `glz::merge` internally so unknown keys will be written at the end of object.
+
+## Example
+
+```c++
+struct my_struct
+{
+  std::string hello;
+  std::string zzz;
+  std::map<glz::sv, glz::raw_json> extra; // store key/raw_json in extra map
+
+  // with methods
+  // void my_unknown_read(const glz::sv& key, const glz::raw_json& value) { 
+  //     extra[key] = value;
+  // };
+  // std::map<glz::sv, glz::raw_json> my_unknown_write() const {
+  //    return extra;
+  // }
+
+};
+
+template <>
+struct glz::meta<my_struct> {
+   using T = my_struct;
+   static constexpr auto value = object(
+      "hello", &T::hello,
+      "zzz", &T::zzz
+   );
+
+   // with members
+   static constexpr auto unknown_write{&T::extra};
+   static constexpr auto unknown_read{&T::extra};
+   // with methods
+   // static constexpr auto unknown_write{&T::my_unknown_write};
+   // static constexpr auto unknown_read{&T::my_unknown_read};
+
+};
+
+// usage
+std::string buffer = R"({"hello":"Hello World!","lang":"en","zzz":"zzz"})";
+my_struct s{};
+
+// decodes and retains extra unknown fields (lang) in extra map
+glz::context ctx{};
+glz::read<glz::opts{.error_on_unknown_keys = false}>(s, buffer, ctx);
+
+// update
+s.hello = "Hi !"
+
+// encodes output string
+std::string out{};
+glz::write_json(s, out);
+
+// out ==  {"hello":"Hi !","zzz":"zzz","lang":"en"}
+```
+
+## Example 2 : Known extra type
+
+If the type of extra keys is known, this would work
+
+```c++
+struct my_struct
+{
+  std::string infinite;
+  int zero;
+  std::map<glz::sv, int> numbers;
+};
+
+template <>
+struct glz::meta<my_struct> {
+   using T = my_struct;
+   static constexpr auto value = object(
+      "infinite", &T::infinite,
+      "zero", &T::zero
+   );
+   
+   static constexpr auto unknown_read{&T::extra};
+};
+
+// usage
+// note extra keys (one, two) are of type int
+std::string buffer = R"({"inf":"infinite","zero":0,"one":1,"two":2})";
+my_struct s{};
+
+// decodes and retains extra unknown fields (lang) in extra map
+glz::context ctx{};
+glz::read<glz::opts{.error_on_unknown_keys = false}>(s, buffer, ctx);
+```
+
+

include/glaze/core/meta.hpp

@@ -68,6 +68,12 @@ namespace glz
 
       template <class T>
       concept glaze_t = requires { meta<std::decay_t<T>>::value; } || local_meta_t<std::decay_t<T>>;
+
+      template <class T>
+      concept has_unknown_writer = requires { meta<T>::unknown_write; } || requires { T::glaze::unknown_write; };
+
+      template <class T>
+      concept has_unknown_reader = requires { meta<T>::unknown_read; } || requires { T::glaze::unknown_read; };
    }
 
    struct empty
@@ -123,6 +129,38 @@ namespace glz
    template <class T>
    using remove_meta_wrapper_t = typename remove_meta_wrapper<T>::type;
 
+   template <class T>
+   inline constexpr auto meta_unknown_write_v = [] {
+      if constexpr (detail::local_meta_t<T>) {
+         return T::glaze::unknown_write;
+      }
+      else if constexpr (detail::global_meta_t<T>) {
+         return meta<T>::unknown_write;
+      }
+      else {
+         return empty{};
+      }
+   }();
+
+   template <class T>
+   using meta_unknown_write_t = std::decay_t<decltype(meta_unknown_write_v<std::decay_t<T>>)>;
+
+   template <class T>
+   inline constexpr auto meta_unknown_read_v = [] {
+      if constexpr (detail::local_meta_t<T>) {
+         return T::glaze::unknown_read;
+      }
+      else if constexpr (detail::global_meta_t<T>) {
+         return meta<T>::unknown_read;
+      }
+      else {
+         return empty{};
+      }
+   }();
+
+   template <class T>
+   using meta_unknown_read_t = std::decay_t<decltype(meta_unknown_read_v<std::decay_t<T>>)>;
+
    template <class T>
    concept named = requires { meta<T>::name; } || requires { T::glaze::name; };
 

include/glaze/core/opts.hpp

@@ -36,6 +36,7 @@ namespace glz
       bool closing_handled = false; // the closing character has been handled
       bool ws_handled = false; // whitespace has already been parsed
       bool no_header = false; // whether or not a binary header is needed
+      bool write_unknown = true; // whether to write unkwown fields
    };
 
    template <opts Opts>
@@ -109,4 +110,20 @@ namespace glz
 
    template <opts Opts, auto member_ptr>
    inline constexpr auto opt_false = opt_off<Opts, member_ptr>();
+
+   template <opts Opts>
+   constexpr auto write_unknown_off()
+   {
+      opts ret = Opts;
+      ret.write_unknown = false;
+      return ret;
+   }
+
+   template <opts Opts>
+   constexpr auto write_unknown_on()
+   {
+      opts ret = Opts;
+      ret.write_unknown = true;
+      return ret;
+   }
 }

include/glaze/json/read.hpp

@@ -78,8 +78,61 @@ namespace glz
                }
             }
          }
+
+         template <auto Opts, class T, is_context Ctx, class It0, class It1>
+         GLZ_ALWAYS_INLINE static void handle_unknown(const glz::sv& key, T&& value, Ctx&& ctx, It0&& it, It1&& end) noexcept
+         {            
+            using ValueType = std::decay_t<decltype(value)>;
+            if constexpr (detail::has_unknown_reader<ValueType>) {
+               constexpr auto& reader = meta_unknown_read_v<ValueType>;
+               using ReaderType = meta_unknown_read_t<ValueType>;
+               if constexpr (std::is_member_object_pointer_v<ReaderType>)
+               {
+                  using MemberType = typename member_value<ReaderType>::type;
+                  if constexpr (detail::map_subscriptable<MemberType>)
+                  {
+                     read<json>::op<Opts>((value.*reader)[key], ctx, it, end);
+                  }
+                  else
+                  {
+                     static_assert(false_v<T>, "target must have subscript operator");
+                  }
+               }
+               else if constexpr (std::is_member_function_pointer_v<ReaderType>)
+               {
+                  using ReturnType = typename return_type<ReaderType>::type;
+                  if constexpr (std::is_void_v<ReturnType>) {
+                     using TupleType = typename inputs_as_tuple<ReaderType>::type;
+                     if constexpr (std::tuple_size_v<TupleType> == 2) {
+                        std::decay_t<std::tuple_element_t<1, TupleType>> input{};
+                        read<json>::op<Opts>(input, ctx, it, end);
+                        if (bool(ctx.error)) [[unlikely]]
+                           return;
+                        (value.*reader)(key, input);
+                     }
+                     else 
+                     {
+                        static_assert(false_v<T>, "method must have 2 args");
+                     }
+                  }
+                  else 
+                  {
+                     static_assert(false_v<T>, "method must have void return");
+                  }
+               }
+               else
+               {
+                  static_assert(false_v<T>, "unknown_read type not handled");
+               }
+            }
+            else
+            {
+               skip_value<Opts>(ctx, it, end);
+            }
+         }
       };
 
+
       template <glaze_value_t T>
       struct from_json<T>
       {
@@ -1572,7 +1625,7 @@ namespace glz
                      if (bool(ctx.error)) [[unlikely]]
                         return;
 
-                     skip_value<Opts>(ctx, it, end);
+                     read<json>::handle_unknown<Opts>(key, value, ctx, it, end);
                      if (bool(ctx.error)) [[unlikely]]
                         return;
                   }
@@ -1624,7 +1677,7 @@ namespace glz
                            if (bool(ctx.error)) [[unlikely]]
                               return;
 
-                           skip_value<Opts>(ctx, it, end);
+                           read<json>::handle_unknown<Opts>(key, value, ctx, it, end);
                            if (bool(ctx.error)) [[unlikely]]
                               return;
                         }
@@ -1635,7 +1688,7 @@ namespace glz
                         if (bool(ctx.error)) [[unlikely]]
                            return;
 
-                        skip_value<Opts>(ctx, it, end);
+                        read<json>::handle_unknown<Opts>(key, value, ctx, it, end);
                         if (bool(ctx.error)) [[unlikely]]
                            return;
                      }

include/glaze/json/write.hpp

@@ -933,8 +933,36 @@ namespace glz
          requires glaze_object_t<T>
       struct to_json<T>
       {
+         template <auto Options, class V>
+         GLZ_FLATTEN static void op(V&& value, is_context auto&& ctx, auto&& b, auto&& ix) noexcept
+         {
+            using ValueType = std::decay_t<V>;
+            if constexpr (detail::has_unknown_writer<ValueType> && Options.write_unknown) {
+               constexpr auto& writer = meta_unknown_write_v<ValueType>;
+
+               using WriterType = meta_unknown_write_t<ValueType>;
+               if constexpr (std::is_member_object_pointer_v<WriterType>)
+               {
+                  write<json>::op<write_unknown_off<Options>()>(glz::merge{value, value.*writer}, ctx, b, ix);
+               }
+               else if constexpr (std::is_member_function_pointer_v<WriterType>)
+               {
+                  write<json>::op<write_unknown_off<Options>()>(glz::merge{value, (value.*writer)()}, ctx, b, ix);
+               }
+               else
+               {
+                  static_assert(false_v<T>, "unknown_write type not handled");
+               }
+            }
+            else
+            {
+               op_base<write_unknown_on<Options>()>(std::forward<V>(value), ctx, b, ix);
+            }
+         }
+
+         // handles glaze_object_t without extra unknown fields
          template <auto Options>
-         GLZ_FLATTEN static void op(auto&& value, is_context auto&& ctx, auto&& b, auto&& ix) noexcept
+         GLZ_FLATTEN static void op_base(auto&& value, is_context auto&& ctx, auto&& b, auto&& ix) noexcept
          {
             if constexpr (!Options.opening_handled) {
                dump<'{'>(b, ix);