Skip to content

Commit

Permalink
Add support to maps
Browse files Browse the repository at this point in the history
  • Loading branch information
@AntoineGagne
AntoineGagne committed Jun 28, 2024
1 parent 0a5affb commit 71d24ab
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 15 deletions.
37 changes: 30 additions & 7 deletions src/parthenon_decode.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,9 @@ object(Binary, Object, Nexts, Schema, Options) ->

-spec object_key(binary(), Buffer :: binary(), object(), [next()], schema(), decode_options()) ->
value().
object_key(<<$=, Rest/binary>>, Key, Object, Nexts, Schema, Options) ->
Next = {object_value, parthenon_utils:lightweight_trim(Key), Object, Schema, Options},
object_key(<<$=, Rest/binary>>, RawKey, Object, Nexts, Schema, Options) ->
Key = encode_key(parthenon_utils:lightweight_trim(RawKey), Schema, Options),
Next = {object_value, Key, Object, Schema, Options},
whitespace(Rest, Next, Nexts);
object_key(<<$,, Rest/binary>>, _Key, Object, Nexts, Schema, Options) ->
object_key(Rest, <<>>, Object, Nexts, Schema, Options);
Expand All @@ -127,26 +128,27 @@ object_key(<<Character, Rest/binary>>, Buffer, Object, Nexts, Schema, Options) -
object_value(<<$=, Rest/binary>>, Key, undefined, Buffer, Object, Nexts, Schema, Options) ->
object_value(Rest, Key, undefined, <<Buffer/binary, $=>>, Object, Nexts, Schema, Options);
object_value(<<$=, Rest/binary>>, Key, LastComma, Buffer, Object, Nexts, Schema, Options) ->
Encoder = wrap_encoder(maps:get(Key, Schema, fun identity/2)),
Encoder = wrap_encoder(value_encoder(Key, Schema)),
Value = binary:part(Buffer, 0, LastComma - 1),
NewKey = parthenon_utils:lightweight_trim(
RawKey = parthenon_utils:lightweight_trim(
binary:part(Buffer, LastComma, byte_size(Buffer) - LastComma)
),
NewKey = encode_key(RawKey, Schema, Options),
EncodedValue = Encoder(Value, Options#decode_options.schema_options),
NewObject = update_object(Key, EncodedValue, Object, Options),
whitespace(Rest, {object_value, NewKey, NewObject, Schema, Options}, Nexts);
object_value(<<$[, Rest/binary>>, Key, _, <<>>, Object, Nexts, Schema, Options) ->
Encoder = maps:get(Key, Schema, fun identity/2),
Encoder = value_encoder(Key, Schema),
Current = {list, [], Encoder, Options},
Next = {object, Key, Object, Schema, Options},
whitespace(Rest, Current, [Next | Nexts]);
object_value(<<${, Rest/binary>>, Key, _, <<>>, Object, Nexts, Schema, Options) ->
Encoder = maps:get(Key, Schema, #{}),
Encoder = value_encoder(Key, Schema, #{}),
Current = {object_key, make_object(Options), Encoder, Options},
Next = {object, Key, Object, Schema, Options},
whitespace(Rest, Current, [Next | Nexts]);
object_value(<<$}, Rest/binary>>, Key, _, Buffer, Object, Nexts, Schema, Options) ->
Encoder = wrap_encoder(maps:get(Key, Schema, fun identity/2)),
Encoder = wrap_encoder(value_encoder(Key, Schema)),
Encoded = Encoder(Buffer, Options#decode_options.schema_options),
NewObject = update_object(Key, Encoded, Object, Options),
next(Rest, NewObject, Nexts);
Expand All @@ -160,15 +162,23 @@ object_value(<<Character, Rest/binary>>, Key, LastComma, Buffer, Object, Nexts,

-spec list(binary(), list(), Buffer :: binary(), [next()], schema(), decode_options()) ->
value().
list(<<$], Rest/binary>>, List, _Buffer, Nexts, {map, _, _}, _Options) ->
next(Rest, lists:reverse(List), Nexts);
list(<<$], Rest/binary>>, List, _Buffer, Nexts, {map_array, _}, _Options) ->
next(Rest, lists:reverse(List), Nexts);
list(<<$], Rest/binary>>, List, Buffer, Nexts, Encoder, Options) ->
Encoded = Encoder(Buffer, Options#decode_options.schema_options),
NewList = lists:reverse([Encoded | List]),
next(Rest, NewList, Nexts);
list(<<${, Rest/binary>>, List, _Buffer, Nexts, Encoder = {map, _, _}, Options) ->
Next = {list, List, Encoder, Options},
object(Rest, make_object(Options), [Next | Nexts], Encoder, Options);
list(<<${, Rest/binary>>, List, _Buffer, Nexts, {map_array, Encoder}, Options) ->
Next = {list, List, {map_array, Encoder}, Options},
object(Rest, make_object(Options), [Next | Nexts], Encoder, Options);
list(<<$,, Rest/binary>>, List, _Buffer, Nexts, Encoder = {map, _, _}, Options) ->
Next = {list, List, Encoder, Options},
whitespace(Rest, Next, Nexts);
list(<<$,, Rest/binary>>, List, _Buffer, Nexts, Encoder = {map_array, _}, Options) ->
Next = {list, List, Encoder, Options},
whitespace(Rest, Next, Nexts);
Expand Down Expand Up @@ -217,6 +227,19 @@ wrap_encoder(Fun) ->
Fun(Value, Options)
end.

encode_key(Raw, {map, KeyEncoder, _}, Options) ->
KeyEncoder(Raw, Options);
encode_key(Key, _Schema, _Options) ->
Key.

value_encoder(Key, Schema) ->
value_encoder(Key, Schema, fun identity/2).

value_encoder(_, {map, _, ValueEncoder}, _Default) ->
ValueEncoder;
value_encoder(Key, Schema, Default) ->
maps:get(Key, Schema, Default).

-spec make_object(decode_options()) -> object().
make_object(#decode_options{object_format = maps}) ->
#{};
Expand Down
2 changes: 2 additions & 0 deletions src/parthenon_schema_lexer.xrl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ word_type("struct") ->
reserved;
word_type("array") ->
reserved;
word_type("map") ->
reserved;
word_type("tinyint") ->
encoding;
word_type("smallint") ->
Expand Down
18 changes: 16 additions & 2 deletions src/parthenon_schema_parser.yrl
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Nonterminals schema struct_root struct_list struct_list_elements struct_element list encoder.
Nonterminals schema struct_root struct_list struct_list_elements struct_element map_root list encoder.

Terminals '<' '>' ',' ':' 'struct' 'array' word encoding.
Terminals '<' '>' ',' ':' 'struct' 'array' 'map' word encoding.

Rootsymbol schema.

Expand All @@ -23,8 +23,12 @@ encoder -> list : '$1'.

encoder -> struct_root : '$1'.

encoder -> map_root : '$1'.

list -> 'array' '<' encoder '>' : create_encoder({list, '$3'}).

map_root -> 'map' '<' encoding ',' encoder '>' : create_encoder({map, '$3', '$5'}).

Erlang code.

-include("parthenon.hrl").
Expand All @@ -43,8 +47,17 @@ create_encoder({encoding, _Line, Encoding}) ->
with_null_as_undefined(with_lightweight_trim(Encoder));
create_encoder({list, Encoder}) when is_map(Encoder) ->
{map_array, Encoder};
create_encoder({list, Encoder = {map, _, _}}) ->
Encoder;
create_encoder({list, Encoder}) ->
with_null_as_undefined(Encoder);
create_encoder({map, {encoding, _, KeyEncoding}, {encoding, _, ValueEncoding}})
when is_map(ValueEncoding) ->
KeyEncoder = to_encoder(KeyEncoding),
{map, KeyEncoder, ValueEncoding};
create_encoder({map, {encoding, _, KeyEncoding}, ValueEncoder}) ->
KeyEncoder = to_encoder(KeyEncoding),
{map, KeyEncoder, ValueEncoder};
create_encoder(Unknown) ->
throw({unknown_encoding, Unknown}).

Expand All @@ -63,6 +76,7 @@ with_lightweight_trim(F) ->
fun(Binary, Options) ->
F(parthenon_utils:lightweight_trim(Binary), Options)
end.

to_encoder(tinyint) ->
fun(Value, _Options) -> binary_to_integer(Value) end;
to_encoder(smallint) ->
Expand Down
54 changes: 53 additions & 1 deletion test/parthenon_decode_SUITE.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,18 @@
-define(A_THIRD_SCHEMA_NAME, schema_3).
-define(A_FOURTH_SCHEMA_NAME, schema_4).
-define(A_FIFTH_SCHEMA_NAME, schema_5).
-define(A_SIXTH_SCHEMA_NAME, schema_6).
-define(A_SEVENTH_SCHEMA_NAME, schema_7).
-define(AN_EIGHT_SCHEMA_NAME, schema_8).
-define(ANOTHER_SCHEMA,
<<"struct<a: int, b: string, c: string, d: int>">>
).
-define(A_THIRD_SCHEMA, <<"array<int>">>).
-define(A_FOURTH_SCHEMA, <<"array<struct<a: int, b: string>>">>).
-define(A_FIFTH_SCHEMA, <<"struct<a: int, b: array<struct<c: int, d: string>>>">>).
-define(A_SIXTH_SCHEMA, <<"struct<a: map<boolean, struct<b: array<int>>>>">>).
-define(A_SEVENTH_SCHEMA, <<"struct<a: map<int, boolean>, b: map<string, string>>">>).
-define(AN_EIGHT_SCHEMA, <<"struct<a: array<map<int, boolean>>>">>).

all() ->
[
Expand All @@ -34,6 +40,10 @@ init_per_suite(Config) ->
ok = parthenon_schema_server:add_schema(?A_THIRD_SCHEMA_NAME, ?A_THIRD_SCHEMA),
ok = parthenon_schema_server:add_schema(?A_FOURTH_SCHEMA_NAME, ?A_FOURTH_SCHEMA),
ok = parthenon_schema_server:add_schema(?A_FIFTH_SCHEMA_NAME, ?A_FIFTH_SCHEMA),
ok = parthenon_schema_server:add_schema(?A_FIFTH_SCHEMA_NAME, ?A_FIFTH_SCHEMA),
ok = parthenon_schema_server:add_schema(?A_SIXTH_SCHEMA_NAME, ?A_SIXTH_SCHEMA),
ok = parthenon_schema_server:add_schema(?A_SEVENTH_SCHEMA_NAME, ?A_SEVENTH_SCHEMA),
ok = parthenon_schema_server:add_schema(?AN_EIGHT_SCHEMA_NAME, ?AN_EIGHT_SCHEMA),
[{schema_server_pid, Pid} | Config].

end_per_suite(Config) ->
Expand All @@ -57,7 +67,10 @@ groups() ->
can_decode_top_level_list,
can_decode_top_level_struct_list,
can_handle_struct_ending_with_array_with_nested_struct,
can_specify_different_null_values
can_specify_different_null_values,
can_decode_map,
can_decode_struct_with_map_from_boolean_to_struct_with_list,
can_decode_list_containing_maps
]}
].

Expand Down Expand Up @@ -234,6 +247,45 @@ can_specify_different_null_values(_Config) ->
)
).

can_decode_map(_Config) ->
?assertEqual(
{ok, #{
a => #{1 => true, 2 => false, 3 => null},
b => #{
<<"test_42">> => <<"bar">>,
<<"application/json">> => <<"test=1">>,
<<"test_43">> => null
}
}},
parthenon_decode:try_decode(
?A_SEVENTH_SCHEMA_NAME,
<<"{a={1=true, 2=false, 3=null}, b={test_42=bar, application/json=test=1, test_43=null}}">>,
[
{schema_options, [{null_as, null}]}
]
)
).

can_decode_struct_with_map_from_boolean_to_struct_with_list(_Config) ->
?assertEqual(
{ok, #{a => #{true => #{b => [1, null, 2]}, false => #{b => [3, 4, 5]}}}},
parthenon_decode:try_decode(
?A_SIXTH_SCHEMA_NAME, <<"{a={true={b=[1, null, 2]}, false={b=[3, 4, 5]}}}">>, [
{schema_options, [{null_as, null}]}
]
)
).

can_decode_list_containing_maps(_Config) ->
?assertEqual(
{ok, #{a => [#{1 => true, 2 => false}, #{3 => null, 4 => false}]}},
parthenon_decode:try_decode(
?AN_EIGHT_SCHEMA_NAME, <<"{a=[{1=true, 2=false}, {3=null, 4=false}]}">>, [
{schema_options, [{null_as, null}]}
]
)
).

%%%===================================================================
%%% Internal functions
%%%===================================================================
57 changes: 52 additions & 5 deletions test/parthenon_schema_SUITE.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,14 @@ contain_correct_encoder(_Config) ->
<<
"struct<boolean_: boolean, int_: int, integer_: integer, double_: double, big_integer: bigint, "
"string_: string, tiny_integer: tinyint, small_integer: smallint, integer_: integer, float_: float,"
"int_list: array<integer>, integer_list: array<integer>, boolean_list: array<boolean>, "
"int_list: array<int>, integer_list: array<integer>, boolean_list: array<boolean>, "
"double_list: array<double>, big_integer_list: array<bigint>,string_list: array<string>, "
"tiny_integer_list: array<tinyint>, small_integer_list: array<smallint>, float_list: array<float>>"
"tiny_integer_list: array<tinyint>, small_integer_list: array<smallint>, float_list: array<float>, "
"int_map: map<int, int>, integer_map: map<integer, integer>, boolean_map: map<boolean, boolean>, "
"double_map: map<double, double>, big_integer_map: map<bigint, bigint>,string_map: map<string, string>, "
"tiny_integer_map: map<tinyint, tinyint>, small_integer_map: map<smallint, smallint>, "
"float_map: map<float, float>, integer_list_map: map<boolean, array<int>>, "
"struct_map: map<integer, struct<foo: int>>>"
>>
),

Expand All @@ -105,6 +110,7 @@ contain_correct_encoder(_Config) ->
?assertEqual(1.0, apply_encoder(<<"float_">>, <<"1.0">>, Schema)),
?assertEqual(100, apply_encoder(<<"big_integer">>, <<"100">>, Schema)),
?assertEqual(<<"test">>, apply_encoder(<<"string_">>, <<"test">>, Schema)),

?assertEqual([2], apply_list_encoder(<<"int_list">>, [<<"2">>], Schema)),
?assertEqual([2], apply_list_encoder(<<"integer_list">>, [<<"2">>], Schema)),
?assertEqual([2], apply_list_encoder(<<"tiny_integer_list">>, [<<"2">>], Schema)),
Expand All @@ -115,7 +121,43 @@ contain_correct_encoder(_Config) ->
?assertEqual([2.0], apply_list_encoder(<<"double_list">>, [<<"2.0">>], Schema)),
?assertEqual([2.0], apply_list_encoder(<<"float_list">>, [<<"2.0">>], Schema)),
?assertEqual([101], apply_list_encoder(<<"big_integer_list">>, [<<"101">>], Schema)),
?assertEqual([<<"test_2">>], apply_list_encoder(<<"string_list">>, [<<"test_2">>], Schema)).
?assertEqual([<<"test_2">>], apply_list_encoder(<<"string_list">>, [<<"test_2">>], Schema)),

?assertEqual({1, 1}, apply_encoder(<<"int_map">>, {<<"1">>, <<"1">>}, Schema)),
?assertEqual({1, 1}, apply_encoder(<<"integer_map">>, {<<"1">>, <<"1">>}, Schema)),
?assertEqual({1, 1}, apply_encoder(<<"tiny_integer_map">>, {<<"1">>, <<"1">>}, Schema)),
?assertEqual({1, 1}, apply_encoder(<<"small_integer_map">>, {<<"1">>, <<"1">>}, Schema)),
?assertEqual(
{true, false}, apply_encoder(<<"boolean_map">>, {<<"true">>, <<"false">>}, Schema)
),
?assertEqual(
{2.0, 2.0}, apply_encoder(<<"double_map">>, {<<"2.0">>, <<"2.0">>}, Schema)
),
?assertEqual(
{101, 101}, apply_encoder(<<"big_integer_map">>, {<<"101">>, <<"101">>}, Schema)
),
?assertEqual(
{<<"test_key">>, <<"test_value">>},
apply_encoder(<<"string_map">>, {<<"test_key">>, <<"test_value">>}, Schema)
),
?assertEqual(
{2.0, 2.0},
apply_encoder(<<"float_map">>, {<<"2.0">>, <<"2.0">>}, Schema)
),

{map, BooleanKeyEncoder, IntEncoder} = maps:get(<<"integer_list_map">>, Schema),
?assertEqual(
{true, [2]},
{BooleanKeyEncoder(<<"true">>, ?SOME_OPTIONS), [
IntEncoder(V, ?SOME_OPTIONS)
|| V <- [<<"2">>]
]}
),
{map, IntegerEncoder, StructEncoder} = maps:get(<<"struct_map">>, Schema),
?assertEqual(
{1, 4},
{IntegerEncoder(<<"1">>, ?SOME_OPTIONS), apply_encoder(<<"foo">>, <<"4">>, StructEncoder)}
).

can_parse_complex_schema(Config) ->
?assertMatch({ok, _}, parthenon_schema:create(?config(raw_schema, Config))).
Expand Down Expand Up @@ -158,8 +200,13 @@ return_undefined_on_null_values(_Config) ->
%%%===================================================================

apply_encoder(Field, Value, Schema) ->
Encoder = maps:get(Field, Schema),
Encoder(Value, ?SOME_OPTIONS).
case maps:get(Field, Schema) of
{map, KeyEncoder, ValueEncoder} ->
{K, V} = Value,
{KeyEncoder(K, ?SOME_OPTIONS), ValueEncoder(V, ?SOME_OPTIONS)};
Encoder ->
Encoder(Value, ?SOME_OPTIONS)
end.

apply_list_encoder(Field, Value, Schema) ->
Encoder = maps:get(Field, Schema),
Expand Down

0 comments on commit 71d24ab

Please sign in to comment.