From 149cc5fc3cf76d6d6924ae323c0e6d6f468b96e1 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 20 Jun 2024 21:49:14 +0100 Subject: [PATCH 1/4] re #442: add tests --- test/test_serialize.cpp | 166 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/test/test_serialize.cpp b/test/test_serialize.cpp index 9866b5b3..8492e06f 100644 --- a/test/test_serialize.cpp +++ b/test/test_serialize.cpp @@ -638,6 +638,172 @@ TEST(deserialize, issue434_3) } +template +void test442(csubstr input, csubstr expected, NodeType style_flag) +{ + // as a seq member + { + const std::string input_str = formatrs("- {}", input); + const std::string expected_input_str = formatrs("- {}\n", input); + const std::string expected_str = formatrs("- {}\n", expected); + const Tree tree = parse_in_arena(to_csubstr(input_str)); + EXPECT_TRUE(tree[0].type_has_all(style_flag)); + EXPECT_EQ(expected_input_str, emitrs_yaml(tree)); + T obj = {}; // T is a scalar type like int, char, double, etc. + tree[0] >> obj; + Tree out_tree; + out_tree.rootref() |= SEQ; + out_tree[0] << obj; + out_tree[0].set_val_style(style_flag); + EXPECT_EQ(expected_str, emitrs_yaml(out_tree)); + } + // as a map member + { + const std::string input_str = formatrs("val: {}", input); + const std::string expected_input_str = formatrs("val: {}\n", input); + const std::string expected_str = formatrs("val: {}\n", expected); + const Tree tree = parse_in_arena(to_csubstr(input_str)); + EXPECT_TRUE(tree["val"].type_has_all(style_flag)); + EXPECT_EQ(expected_input_str, emitrs_yaml(tree)); + T obj = {}; // T is a scalar type like int, char, double, etc. + tree["val"] >> obj; + Tree out_tree; + out_tree.rootref() |= MAP; + out_tree["val"] << obj; + out_tree["val"].set_val_style(style_flag); + EXPECT_EQ(expected_str, emitrs_yaml(out_tree)); + } + // as a doc scalar + { + const std::string expected_input_str = formatrs("{}\n", input); + const std::string expected_str = formatrs("{}\n", expected); + const Tree tree = parse_in_arena(input); + EXPECT_TRUE(tree.rootref().type_has_all(style_flag)); + EXPECT_EQ(expected_input_str, emitrs_yaml(tree)); + T obj = {}; // T is a scalar type like int, char, double, etc. + tree.rootref() >> obj; + Tree out_tree; + out_tree.rootref() << obj; + out_tree.rootref().set_val_style(style_flag); + EXPECT_EQ(expected_str, emitrs_yaml(out_tree)); + } +} +TEST(serialize, issue442_00) +{ + test442("123", "123", VAL_PLAIN); +} +TEST(serialize, issue442_01) +{ + test442("-123", "-123", VAL_PLAIN); +} +TEST(serialize, issue442_02) +{ + test442("+123", "123", VAL_PLAIN); +} +TEST(serialize, issue442_10) +{ + test442("2.35e-10", "2.35e-10", VAL_PLAIN); +} +TEST(serialize, issue442_11) +{ + test442("-2.35e-10", "-2.35e-10", VAL_PLAIN); +} +TEST(serialize, issue442_12) +{ + test442("+2.35e-10", "2.35e-10", VAL_PLAIN); +} +TEST(serialize, issue442_20) +{ + test442("2.35e-10", "2.35e-10", VAL_PLAIN); +} +TEST(serialize, issue442_21) +{ + test442("-2.35e-10", "-2.35e-10", VAL_PLAIN); +} +TEST(serialize, issue442_22) +{ + test442("+2.35e-10", "2.35e-10", VAL_PLAIN); +} +TEST(serialize, issue442_30) +{ + test442("'a'", "'a'", VAL_SQUO); +} +TEST(serialize, issue442_31) +{ + test442("' '", "' '", VAL_SQUO); +} +TEST(serialize, issue442_40) +{ + test442("\"a\"", "\"a\"", VAL_DQUO); +} +TEST(serialize, issue442_41) +{ + test442("\" \"", "\" \"", VAL_DQUO); +} +TEST(serialize, issue442_50) +{ + test442("a", "a", VAL_PLAIN); +} +TEST(serialize, issue442_60) +{ + EXPECT_TRUE(scalar_style_query_plain("123")); + EXPECT_TRUE(scalar_style_query_plain("-123")); + EXPECT_TRUE(scalar_style_query_plain("+123")); + EXPECT_EQ(scalar_style_choose("123"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-123"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+123"), SCALAR_PLAIN); + { + Tree tree; + tree.rootref() << "123"; + EXPECT_EQ(std::string("123\n"), emitrs_yaml(tree)); + } + { + Tree tree; + tree.rootref() << "-123"; + EXPECT_EQ(std::string("-123\n"), emitrs_yaml(tree)); + } + { + Tree tree; + tree.rootref() << 123; + EXPECT_EQ(std::string("123\n"), emitrs_yaml(tree)); + } + { + Tree tree; + tree.rootref() << -123; + EXPECT_EQ(std::string("-123\n"), emitrs_yaml(tree)); + } +} +TEST(serialize, issue442_61) +{ + EXPECT_TRUE(scalar_style_query_plain("2.35e-10")); + EXPECT_TRUE(scalar_style_query_plain("-2.35e-10")); + EXPECT_TRUE(scalar_style_query_plain("+2.35e-10")); + EXPECT_EQ(scalar_style_choose("2.35e-10"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-2.35e-10"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+2.35e-10"), SCALAR_PLAIN); + { + Tree tree; + tree.rootref() << 2.35e-10; + EXPECT_EQ(std::string("2.35e-10\n"), emitrs_yaml(tree)); + } + { + Tree tree; + tree.rootref() << -2.35e-10; + EXPECT_EQ(std::string("-2.35e-10\n"), emitrs_yaml(tree)); + } + { + Tree tree; + tree.rootref() << "2.35e-10"; + EXPECT_EQ(std::string("2.35e-10\n"), emitrs_yaml(tree)); + } + { + Tree tree; + tree.rootref() << "-2.35e-10"; + EXPECT_EQ(std::string("-2.35e-10\n"), emitrs_yaml(tree)); + } +} + + //------------------------------------------- // this is needed to use the test case library Case const* get_case(csubstr /*name*/) From b2e920910dcfc4b53a425f1b63db2a150c6df83b Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 20 Jun 2024 21:48:53 +0100 Subject: [PATCH 2/4] re #442: add quickstart code for overflow detection --- changelog/current.md | 4 ++++ ext/c4core | 2 +- samples/quickstart.cpp | 28 ++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/changelog/current.md b/changelog/current.md index 362885e4..2cde82e4 100644 --- a/changelog/current.md +++ b/changelog/current.md @@ -1,3 +1,7 @@ ## Fixes +- Fix [#442](https://github.com/biojppm/rapidyaml/issues/442) ([PR#443](https://github.com/biojppm/rapidyaml/pull/443)): + - Ensure leading `+` is accepted when deserializing numbers. + - Ensure numbers are not quoted by fixing the heuristics in `scalar_style_query_plain()` and `scalar_style_choose()`. + - Add quickstart sample for overflow detection (only of integral types). - Parse engine: cleanup unused macros diff --git a/ext/c4core b/ext/c4core index 1cf2a755..f9e970ac 160000 --- a/ext/c4core +++ b/ext/c4core @@ -1 +1 @@ -Subproject commit 1cf2a755a5853651e42aec99c5ef49bb2ec56cf5 +Subproject commit f9e970ac152034544d13dc3d9d109e17504274b1 diff --git a/samples/quickstart.cpp b/samples/quickstart.cpp index eed235d2..09224c4f 100644 --- a/samples/quickstart.cpp +++ b/samples/quickstart.cpp @@ -2488,6 +2488,34 @@ void sample_fundamental_types() tree["nan" ] >> f; CHECK(std::isnan(f)); tree["nan" ] >> d; CHECK(std::isnan(d)); C4_SUPPRESS_WARNING_GCC_CLANG_POP + + // value overflow detection: + // (for integral types only) + { + // we will be detecting errors below, so we use this sample helper + ScopedErrorHandlerExample err = {}; + ryml::Tree t(err.callbacks()); // instantiate with the error-detecting callbacks + // create a simple tree with an int value + ryml::parse_in_arena(R"({val: 258})", &t); + // by default, overflow is not detected: + uint8_t valu8 = 0; + int8_t vali8 = 0; + t["val"] >> valu8; CHECK(valu8 == 2); // not 257; it wrapped around + t["val"] >> vali8; CHECK(vali8 == 2); // not 257; it wrapped around + // ...but there are facilities to detect overflow + CHECK(ryml::overflows(t["val"].val())); + CHECK(ryml::overflows(t["val"].val())); + CHECK( ! ryml::overflows(t["val"].val())); + // and there is a format helper + CHECK(err.check_error_occurs([&]{ + auto checku8 = ryml::fmt::overflow_checked(valu8); // need to declare the wrapper type before using it with >> + t["val"] >> checku8; // this will cause an error + })); + CHECK(err.check_error_occurs([&]{ + auto checki8 = ryml::fmt::overflow_checked(vali8); // need to declare the wrapper type before using it with >> + t["val"] >> checki8; // this will cause an error + })); + } } From 444f7ff40a0cc4c8f6e0c827d88160fb31d96127 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 20 Jun 2024 22:55:02 +0100 Subject: [PATCH 3/4] re #442: fix style heuristics for numbers --- src/c4/yml/node_type.cpp | 4 ++ test/test_node_type.cpp | 115 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 113 insertions(+), 6 deletions(-) diff --git a/src/c4/yml/node_type.cpp b/src/c4/yml/node_type.cpp index fe4119ca..dac9bd27 100644 --- a/src/c4/yml/node_type.cpp +++ b/src/c4/yml/node_type.cpp @@ -153,6 +153,10 @@ bool scalar_style_query_plain(csubstr s) noexcept else if(s.sub(2).is_number()) return true; } + else if(s.begins_with_any("0123456789.-+") && s.is_number()) + { + return true; + } return s != ':' && ( ! s.begins_with_any("-:?*&,'\"{}[]|>%#@`\r")) // @ and ` are reserved characters && ( ! s.ends_with_any(":#")) diff --git a/test/test_node_type.cpp b/test/test_node_type.cpp index 3ba00ed0..47b55b36 100644 --- a/test/test_node_type.cpp +++ b/test/test_node_type.cpp @@ -129,29 +129,132 @@ TEST(NodeType, type_str) #undef teststr } -TEST(NodeType, scalar_style_choose) -{ - EXPECT_EQ(scalar_style_choose(" \n\t"), SCALAR_DQUO); - EXPECT_EQ(scalar_style_choose("01"), SCALAR_PLAIN); -} - TEST(NodeType, scalar_style_choose_json) { EXPECT_EQ(scalar_style_json_choose("true"), SCALAR_PLAIN); EXPECT_EQ(scalar_style_json_choose("false"), SCALAR_PLAIN); EXPECT_EQ(scalar_style_json_choose("null"), SCALAR_PLAIN); EXPECT_EQ(scalar_style_json_choose("0.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_json_choose("-0.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_json_choose("+0.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_json_choose(".1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_json_choose("+.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_json_choose("-.1"), SCALAR_PLAIN); EXPECT_EQ(scalar_style_json_choose("01"), SCALAR_DQUO); EXPECT_EQ(scalar_style_json_choose("foo"), SCALAR_DQUO); EXPECT_EQ(scalar_style_json_choose("bar"), SCALAR_DQUO); } +TEST(NodeType, scalar_style_choose) +{ + EXPECT_EQ(scalar_style_choose(" \n\t"), SCALAR_DQUO); + EXPECT_EQ(scalar_style_choose("-.inf"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-.INF"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-.034"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-.034x"), SCALAR_SQUO); + EXPECT_EQ(scalar_style_choose("2.35e-10"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-2.35e-10"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+2.35e-10"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0.1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("01"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0x1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0o1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0b1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0x1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0o1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0b1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-01"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0x1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0o1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0b1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0x1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0o1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0b1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+01"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0x1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0o1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0b1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0x1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0o1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0b1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("01"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0X1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0O1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0B1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0X1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0O1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("0B1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-01"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0X1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0O1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0B1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0X1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0O1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("-0B1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+01"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0X1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0O1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0B1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0X1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0O1"), SCALAR_PLAIN); + EXPECT_EQ(scalar_style_choose("+0B1"), SCALAR_PLAIN); +} + TEST(NodeType, scalar_style_query_plain) { EXPECT_TRUE(scalar_style_query_plain("-.inf")); EXPECT_TRUE(scalar_style_query_plain("-.INF")); EXPECT_TRUE(scalar_style_query_plain("-.034")); EXPECT_FALSE(scalar_style_query_plain("-.034x")); + EXPECT_TRUE(scalar_style_query_plain("2.35e-10")); + EXPECT_TRUE(scalar_style_query_plain("-2.35e-10")); + EXPECT_TRUE(scalar_style_query_plain("+2.35e-10")); + EXPECT_TRUE(scalar_style_query_plain("0.1")); + EXPECT_TRUE(scalar_style_query_plain("-0.1")); + EXPECT_TRUE(scalar_style_query_plain("+0.1")); + EXPECT_TRUE(scalar_style_query_plain("01")); + EXPECT_TRUE(scalar_style_query_plain("0x1")); + EXPECT_TRUE(scalar_style_query_plain("0o1")); + EXPECT_TRUE(scalar_style_query_plain("0b1")); + EXPECT_TRUE(scalar_style_query_plain("0x1")); + EXPECT_TRUE(scalar_style_query_plain("0o1")); + EXPECT_TRUE(scalar_style_query_plain("0b1")); + EXPECT_TRUE(scalar_style_query_plain("+01")); + EXPECT_TRUE(scalar_style_query_plain("+0x1")); + EXPECT_TRUE(scalar_style_query_plain("+0o1")); + EXPECT_TRUE(scalar_style_query_plain("+0b1")); + EXPECT_TRUE(scalar_style_query_plain("+0x1")); + EXPECT_TRUE(scalar_style_query_plain("+0o1")); + EXPECT_TRUE(scalar_style_query_plain("+0b1")); + EXPECT_TRUE(scalar_style_query_plain("-01")); + EXPECT_TRUE(scalar_style_query_plain("-0x1")); + EXPECT_TRUE(scalar_style_query_plain("-0o1")); + EXPECT_TRUE(scalar_style_query_plain("-0b1")); + EXPECT_TRUE(scalar_style_query_plain("-0x1")); + EXPECT_TRUE(scalar_style_query_plain("-0o1")); + EXPECT_TRUE(scalar_style_query_plain("-0b1")); + EXPECT_TRUE(scalar_style_query_plain("0X1")); + EXPECT_TRUE(scalar_style_query_plain("0O1")); + EXPECT_TRUE(scalar_style_query_plain("0B1")); + EXPECT_TRUE(scalar_style_query_plain("0X1")); + EXPECT_TRUE(scalar_style_query_plain("0O1")); + EXPECT_TRUE(scalar_style_query_plain("0B1")); + EXPECT_TRUE(scalar_style_query_plain("+01")); + EXPECT_TRUE(scalar_style_query_plain("+0X1")); + EXPECT_TRUE(scalar_style_query_plain("+0O1")); + EXPECT_TRUE(scalar_style_query_plain("+0B1")); + EXPECT_TRUE(scalar_style_query_plain("+0X1")); + EXPECT_TRUE(scalar_style_query_plain("+0O1")); + EXPECT_TRUE(scalar_style_query_plain("+0B1")); + EXPECT_TRUE(scalar_style_query_plain("-01")); + EXPECT_TRUE(scalar_style_query_plain("-0X1")); + EXPECT_TRUE(scalar_style_query_plain("-0O1")); + EXPECT_TRUE(scalar_style_query_plain("-0B1")); + EXPECT_TRUE(scalar_style_query_plain("-0X1")); + EXPECT_TRUE(scalar_style_query_plain("-0O1")); + EXPECT_TRUE(scalar_style_query_plain("-0B1")); } From 8f0d631a997d4046751c5058c724dc414ba18445 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 21 Jun 2024 16:58:52 +0100 Subject: [PATCH 4/4] re #442: ensure leading + is accepted --- src/c4/yml/node.hpp | 30 ++++++++++++++++++++++++------ src/c4/yml/tree.hpp | 4 ++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/c4/yml/node.hpp b/src/c4/yml/node.hpp index ccda50c3..b3f707f2 100644 --- a/src/c4/yml/node.hpp +++ b/src/c4/yml/node.hpp @@ -1604,31 +1604,49 @@ inline void write(NodeRef *n, T const& v) n->set_val_serialized(v); } +namespace detail { +// SFINAE overloads for skipping leading + which cannot be read by the charconv functions +template +C4_ALWAYS_INLINE auto read_skip_plus(csubstr val, T *v) + -> typename std::enable_if::value, bool>::type +{ + if(val.begins_with('+')) + val = val.sub(1); + return from_chars(val, v); +} +template +C4_ALWAYS_INLINE auto read_skip_plus(csubstr val, T *v) + -> typename std::enable_if< ! std::is_arithmetic::value, bool>::type +{ + return from_chars(val, v); +} +} // namespace detail + /** convert the val of a scalar node to a particular type, by * forwarding its val to @ref from_chars(). The full string is * used. * @return false if the conversion failed */ template -typename std::enable_if< ! std::is_floating_point::value, bool>::type -inline read(NodeRef const& n, T *v) +inline auto read(NodeRef const& n, T *v) + -> typename std::enable_if< ! std::is_floating_point::value, bool>::type { csubstr val = n.val(); if(val.empty()) return false; - return from_chars(val, v); + return detail::read_skip_plus(val, v); } /** convert the val of a scalar node to a particular type, by * forwarding its val to @ref from_chars(). The full string is * used. * @return false if the conversion failed */ template -typename std::enable_if< ! std::is_floating_point::value, bool>::type -inline read(ConstNodeRef const& n, T *v) +inline auto read(ConstNodeRef const& n, T *v) + -> typename std::enable_if< ! std::is_floating_point::value, bool>::type { csubstr val = n.val(); if(val.empty()) return false; - return from_chars(val, v); + return detail::read_skip_plus(val, v); } /** convert the val of a scalar node to a floating point type, by diff --git a/src/c4/yml/tree.hpp b/src/c4/yml/tree.hpp index 0dbcb3de..49b1ec7c 100644 --- a/src/c4/yml/tree.hpp +++ b/src/c4/yml/tree.hpp @@ -64,6 +64,10 @@ bool from_chars_float(csubstr buf, T *C4_RESTRICT val) { return true; } + else if(C4_UNLIKELY(buf.begins_with('+'))) + { + return from_chars(buf.sub(1), val); + } else if(C4_UNLIKELY(buf == ".nan" || buf == ".NaN" || buf == ".NAN")) { *val = std::numeric_limits::quiet_NaN();