2

Michael Caisse gave a talk on Spirit X3: https://www.youtube.com/watch?v=xSBWklPLRvw. I've tried to transcribe the presentation to working code, but I have compilation errors that include "No viable overloaded '='" and "No matching function call to 'move_to.' Is my phrase_parse (last line) defined correctly? Are there any obvious errors?

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/container/stable_vector.hpp>    

#include <iostream>
#include <iterator>
#include <fstream>
#include <string>
#include <map>    

namespace client { namespace ast
    {
        namespace x3 = boost::spirit::x3;

        using string_t = std::string;
        using double_t = double;
        using float_t = double;
        using int_t = int64_t;
        using bool_t = bool;
        struct null_t {};

        class value;

        using object_t = std::map<std::string, value>;
        using object_member_t = object_t::value_type;
        using array_t = boost::container::stable_vector<value>;

        class value : public x3::variant<null_t, bool_t, string_t, int_t, double_t, object_t, array_t>
        {
        public:
            using value_type = value;
            using base_type::base_type;
            using base_type::operator=;
            value(null_t val = null_t{}) : base_type(val) {}
            value(char const* val)       : base_type(string_t(val)) {}

            template<typename T>
            value(T val, typename std::enable_if<std::is_floating_point<T>::value>::type) : base_type(double_t{val}) {}

            template<typename T>
            value(T val, typename std::enable_if<std::is_integral<T>::value::type>) : base_type(int_t{val}) {}

        };

        struct json_class;
        using json_type = x3::rule<json_class, value>;
        json_type const json = "json";
        BOOST_SPIRIT_DECLARE(json_type);

        // identifier_class id
        // identifier_type  type
        // identifier_def   rule def
        // identifier       the rule

        struct value_class;
        struct object_class;
        struct member_pair_class;
        struct array_class;

        using value_type = x3::rule<value_class, value>;
        using object_type = x3::rule<object_class, object_t>;
        using member_pair_type = x3::rule<member_pair_class, object_member_t>;
        using array_type = x3::rule<array_class, array_t>;

        value_type const value = "value";
        object_type const object = "object";
        member_pair_type const member_pair = "member_pair";
        array_type const array = "array";

        auto const append = [](auto& ctx){ _val(ctx) += _attr(ctx); };

        using uchar = unsigned char;

        x3::uint_parser<uchar, 16, 4, 4> const hex4 = {};

        auto push_esc = [](auto& ctx)
        {
            auto& utf8 = _val(ctx);
            switch (_attr(ctx))
            {
                case '"' : utf8 += '"';   break;
                case '\\': utf8 += '\\';  break;
                case '/' : utf8 += '/';   break;
                case 'b' : utf8 += '\b';  break;
                case 'f' : utf8 += '\f';  break;
                case 'n' : utf8 += '\n';  break;
                case 'r' : utf8 += '\r';  break;
                case 't' : utf8 += '\t';  break;
            }
        };

        auto push_utf8 = [](auto& ctx)
        {
            typedef std::back_insert_iterator<std::string> insert_iter;
            insert_iter out_iter(_val(ctx));
            boost::utf8_output_iterator<insert_iter> utf8_iter(out_iter);
            *utf8_iter++ = _attr(ctx);
        };

        auto const escape = ('u' > hex4)            [push_utf8]
                          | x3::char_("\"\\/bfnrt") [push_esc];

        auto const char_esc = '\\' > escape;

        auto const double_quoted = x3::lexeme[ '"' > *(char_esc) | (x3::char_("\x20\x21\x23-\x5b\x5d\x7e")) [append]  > '"' ];

        struct unicode_string_class;
        using unicode_string_type = x3::rule<unicode_string_class, std::string>;
        unicode_string_type const unicode_string = "unicode_string";
        auto const unicode_string_def = double_quoted;
        BOOST_SPIRIT_DEFINE(unicode_string);    

        auto const null_value = x3::lit("null") >> x3::attr(null_t{});
        x3::int_parser<int64_t> const int_ = {};
        x3::ascii::bool_type const bool_value = {};

        auto const object_def = x3::lit('{') >> -(member_pair % ',') >> x3::lit('}');    

        auto const member_pair_def = unicode_string >> ':' >> value;

        auto const array_def = x3::lit('[') >> -(value % ',') >> x3::lit(']');

        auto const value_def = null_value | bool_value | object | array | unicode_string
                             | x3::lexeme[!('+' | (-x3::lit('-') >> '0' >> x3::digit)) >> x3::int_ >> !x3::char_(".eE")]
                             | x3::lexeme[!('+' | (-x3::lit('-') >> '0' >> x3::digit)) >> x3::double_ ];

        BOOST_SPIRIT_DEFINE(value, object, member_pair, array);
    }
}    

int main(int argc, char **argv)
{

    namespace x3 = boost::spirit::x3;

    std::string storage; // We will read the contents here.    

    using boost::spirit::x3::ascii::space;
    std::string::const_iterator iter = storage.begin();
    std::string::const_iterator iter_end = storage.end();

    client::ast::object_t o;
    auto const grammar = client::ast::value;

    bool r = phrase_parse(iter, iter_end, grammar, space, o);
}
John Estess
  • 576
  • 10
  • 22
  • https://www.livecoding.tv/sehe/ looking – sehe Dec 10 '15 at 14:38
  • I believe a version of that code is also up on github: https://github.com/cierelabs/json_spirit (I didn't use it when answering the question, see the [recorded livestream](https://www.livecoding.tv/video/fixing-spirit-x3-json-parser-part-1) and [part 2](https://www.livecoding.tv/video/fixing-spirit-x3-json-parser-part-2)) – sehe Dec 10 '15 at 15:49
  • The readme at github mentions Spirit 2 and the code doesn't appear to have x3 headers. I dropped this in XCode and it compiles and seems to work perfectly. I'll give this a proper review when I get back. Thank you! – John Estess Dec 10 '15 at 16:26
  • The missing hyphen was completely my fault. – John Estess Dec 11 '15 at 04:40
  • The livecoding sessions were awesome, but it appears livecoding.tv is no more. – John Estess Dec 22 '22 at 01:37
  • Yup. [They're long gone.](https://stackoverflow.com/users/85371/sehe?tab=profile#:~:text=Sadly%20they%20removed%20all%20of%20the%20few%20hundreds%20hours%20of%20stream%20after%20they%20changed%2Dover%20their%20business%20model%20a%20few%20times.%20There%20will%20be%20quite%20a%20few%20now%2Ddead%20links%20to%20streams%20in%20the%20comments%20at%20my%20answers.%20I%27m%20sorry.) – sehe Dec 22 '22 at 03:15

1 Answers1

4

Okay, I had to iron out quite a few errors/quirks.

Part of my ironing has been clean-ups of things I usually do differently. Here's the working result:

Live On Coliru

#define BOOST_SPIRIT_X3_DEBUG
#include <iostream>
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/container/stable_vector.hpp>    

#include <iterator>
#include <fstream>
#include <string>
#include <map>    

namespace client { 
    namespace x3 = boost::spirit::x3;

    namespace ast
    {
        using string_t = std::string;
        using double_t = double;
        using float_t = double;
        using int_t = int64_t;
        using bool_t = bool;
        struct null_t {};

        class value;

        using object_t        = std::map<std::string, value>;
        using object_member_t = object_t::value_type;
        using member_pair_t   = std::pair<object_t::key_type, object_t::mapped_type>;
        using array_t         = boost::container::stable_vector<value>;

        class value : public x3::variant<null_t, bool_t, string_t, int_t, double_t, object_t, array_t>
        {
        public:
            using value_type = value;
            using base_type::base_type;
            using base_type::operator=;
            value(null_t val = null_t{}) : base_type(val) {}
            value(char const* val)       : base_type(string_t(val)) {}

            template<typename T>
            value(T val, typename std::enable_if<std::is_floating_point<T>::value>::type) : base_type(double_t{val}) {}

            template<typename T>
            value(T val, typename std::enable_if<std::is_integral<T>::value::type>) : base_type(int_t{val}) {}

        };

    }

    namespace parser
    {
        auto const append = [](auto& ctx){ x3::_val(ctx) += x3::_attr(ctx); };

        using uchar = unsigned char;

        x3::uint_parser<uchar, 16, 4, 4> const hex4 = {};

        auto push_esc = [](auto& ctx) {
            auto& utf8 = x3::_val(ctx);
            switch (x3::_attr(ctx))
            {
                case '"' : utf8 += '"';   break;
                case '\\': utf8 += '\\';  break;
                case '/' : utf8 += '/';   break;
                case 'b' : utf8 += '\b';  break;
                case 'f' : utf8 += '\f';  break;
                case 'n' : utf8 += '\n';  break;
                case 'r' : utf8 += '\r';  break;
                case 't' : utf8 += '\t';  break;
            }
        };

        auto push_utf8 = [](auto& ctx) {
            typedef std::back_insert_iterator<std::string> insert_iter;
            insert_iter out_iter(x3::_val(ctx));
            boost::utf8_output_iterator<insert_iter> utf8_iter(out_iter);
            *utf8_iter++ = x3::_attr(ctx);
        };

        auto const escape = ('u' > hex4)            [push_utf8]
                          | x3::char_("\"\\/bfnrt") [push_esc];

        auto const char_esc = '\\' > escape;

        auto const double_quoted = x3::lexeme[ 
                            '"' > *(char_esc | (x3::char_("\x20\x21\x23-\x5b\x5d-\x7e") [append])) > '"' 
                        ];

        auto const unicode_string 
                         = x3::rule<struct unicode_string_class, std::string> { "unicode_string" } 
                         = double_quoted;

        auto const null_value = x3::lit("null") >> x3::attr(ast::null_t{});

        x3::ascii::bool_type const bool_value = {};

        using value_type = x3::rule<struct value_class, ast::value>;
        static value_type const value = "value";

        auto const member_pair = x3::rule<struct member_pair_class, ast::member_pair_t> { "member_pair" }
                          = unicode_string >> ':' >> value;

        auto const object = x3::rule<struct object_class, ast::object_t> { "object" }
                          = x3::lit('{') >> -(member_pair % ',') >> x3::lit('}');    

        auto const array = x3::rule<struct array_class, ast::array_t> { "array" }
                         = x3::lit('[') >> -(value % ',') >> x3::lit(']');

        x3::real_parser<double, x3::strict_real_policies<double> > const double_ = {};
        x3::int_parser<int64_t> const int_                                       = {};

        auto const value_def = null_value | bool_value | object | array | unicode_string | double_ | int_ ;

        BOOST_SPIRIT_DEFINE(value)

        auto const json = x3::skip(x3::ascii::space) [ value ];
    }
}    

int main()
{
    std::string storage = R"({ "check": [ 1,2,3, null, true ], "more": { "nested" : "values" } })";

    client::ast::value o;
    return parse(storage.begin(), storage.end(), client::parser::json, o)? 0 : 255;
}

Outputting:

<value>
<try>{ "check": [ 1,2,3, </try>
<object>
    <try>{ "check": [ 1,2,3, </try>
    <member_pair>
    <try> "check": [ 1,2,3, n</try>
    <unicode_string>
        <try> "check": [ 1,2,3, n</try>
        <success>: [ 1,2,3, null, tru</success>
        <attributes>[c, h, e, c, k]</attributes>
    </unicode_string>
    <value>
        <try> [ 1,2,3, null, true</try>
        <object>
        <try>[ 1,2,3, null, true </try>
        <fail/>
        </object>
        <array>
        <try>[ 1,2,3, null, true </try>
        <value>
            <try> 1,2,3, null, true ]</try>
            <object>
            <try>1,2,3, null, true ],</try>
            <fail/>
            </object>
            <array>
            <try>1,2,3, null, true ],</try>
            <fail/>
            </array>
            <unicode_string>
            <try>1,2,3, null, true ],</try>
            <fail/>
            </unicode_string>
            <success>,2,3, null, true ], </success>
            <attributes>1</attributes>
        </value>
        <value>
            <try>2,3, null, true ], "</try>
            <object>
            <try>2,3, null, true ], "</try>
            <fail/>
            </object>
            <array>
            <try>2,3, null, true ], "</try>
            <fail/>
            </array>
            <unicode_string>
            <try>2,3, null, true ], "</try>
            <fail/>
            </unicode_string>
            <success>,3, null, true ], "m</success>
            <attributes>2</attributes>
        </value>
        <value>
            <try>3, null, true ], "mo</try>
            <object>
            <try>3, null, true ], "mo</try>
            <fail/>
            </object>
            <array>
            <try>3, null, true ], "mo</try>
            <fail/>
            </array>
            <unicode_string>
            <try>3, null, true ], "mo</try>
            <fail/>
            </unicode_string>
            <success>, null, true ], "mor</success>
            <attributes>3</attributes>
        </value>
        <value>
            <try> null, true ], "more</try>
            <success>, true ], "more": { </success>
            <attributes></attributes>
        </value>
        <value>
            <try> true ], "more": { "</try>
            <success> ], "more": { "neste</success>
            <attributes>1</attributes>
        </value>
        <success>, "more": { "nested"</success>
        <attributes>[1, 2, 3, , 1]</attributes>
        </array>
        <success>, "more": { "nested"</success>
        <attributes>[1, 2, 3, , 1]</attributes>
    </value>
    <success>, "more": { "nested"</success>
    <attributes>[[c, h, e, c, k], [1, 2, 3, , 1]]</attributes>
    </member_pair>
    <member_pair>
    <try> "more": { "nested" </try>
    <unicode_string>
        <try> "more": { "nested" </try>
        <success>: { "nested" : "valu</success>
        <attributes>[m, o, r, e]</attributes>
    </unicode_string>
    <value>
        <try> { "nested" : "value</try>
        <object>
        <try>{ "nested" : "values</try>
        <member_pair>
            <try> "nested" : "values"</try>
            <unicode_string>
            <try> "nested" : "values"</try>
            <success> : "values" } }</success>
            <attributes>[n, e, s, t, e, d]</attributes>
            </unicode_string>
            <value>
            <try> "values" } }</try>
            <object>
                <try>"values" } }</try>
                <fail/>
            </object>
            <array>
                <try>"values" } }</try>
                <fail/>
            </array>
            <unicode_string>
                <try>"values" } }</try>
                <success> } }</success>
                <attributes>[v, a, l, u, e, s]</attributes>
            </unicode_string>
            <success> } }</success>
            <attributes>[v, a, l, u, e, s]</attributes>
            </value>
            <success> } }</success>
            <attributes>[[n, e, s, t, e, d], [v, a, l, u, e, s]]</attributes>
        </member_pair>
        <success> }</success>
        <attributes>[[[n, e, s, t, e, d], [v, a, l, u, e, s]]]</attributes>
        </object>
        <success> }</success>
        <attributes>[[[n, e, s, t, e, d], [v, a, l, u, e, s]]]</attributes>
    </value>
    <success> }</success>
    <attributes>[[m, o, r, e], [[[n, e, s, t, e, d], [v, a, l, u, e, s]]]]</attributes>
    </member_pair>
    <success></success>
    <attributes>[[[c, h, e, c, k], [1, 2, 3, , 1]], [[m, o, r, e], [[[n, e, s, t, e, d], [v, a, l, u, e, s]]]]]</attributes>
</object>
<success></success>
<attributes>[[[c, h, e, c, k], [1, 2, 3, , 1]], [[m, o, r, e], [[[n, e, s, t, e, d], [v, a, l, u, e, s]]]]]</attributes>
</value>

Some notes:

  1. The failing assignment was because the parser rule value results in an object of type value, not object_t...

  2. object_member_t is a pair with a std::string const first_type. Oops. That can not be assigned to either. So, make your own pair-type and use it:

    using member_pair_t   = std::pair<object_t::key_type, object_t::mapped_type>;
    
  3. My g++5 seemed to have trouble doing ADL lookup of _attr and _val if left unqualified. I'm not sure whether that's a compiler issue, or what. I just went with x3::_attr and x3::_val for now

  4. There was a bunch of missing parens in rule defs that made the rules hard to read/check. Added them

  5. unicode_string turned out to be broken (see below)

  6. The int_parser<int64_t> was never used (? huh). Use it :)

  7. The number parsing was not functioning. At the same time it looked overly complicated. I suggest using double_ | int_ where double_ uses the strict policy:

    x3::real_parser<double, x3::strict_real_policies<double> > const double_ = {};
    

Matters of style:

  1. Don't make your caller responsible for choosing a skipper. After all, if you use x3::char_("+8") as the skipper, things will either break or fail to be JSON? I suggest a top-level rule to just introduce the skipper:

    auto const json = x3::skip(x3::ascii::space) [ value ];
    
  2. The test driver can be much less noisy:

    std::string storage = R"({ "check": [ 1,2,3, null, true ], "more": { "nested" : "values" } })";
    
    client::ast::value o;
    bool r = parse(storage.begin(), storage.end(), client::parser::json, o);
    
  3. I removed the noisy _class/_type/_def dance for rules that are not used "exernally" (in this simple code sample, anything that's not used recursively: value)

  4. Use BOOST_SPIRIT_X3_DEBUG to see what's happening :) (this allowed me to spot the error in x3::char_("\x20\x21\x23-\x5b\x5d\x7e"), see the missing dash?)

sehe
  • 374,641
  • 47
  • 450
  • 633
  • 1
    Thanks for sharing this in excruciating and helpful detail! – uchuugaka Jan 14 '17 at 02:37
  • With boost 1.65.1 and gcc 7.2, this gives `/boost/spirit/home/x3/nonterminal/detail/rule.hpp:316:24: error: use of deleted function ‘client::ast::value::value(const client::ast::value&)’ value_type made_attr = make_attribute::call(attr); test14.cc:35:15: note: ‘client::ast::value::value(const client::ast::value&)’ is implicitly declared as deleted because ‘client::ast::value’ declares a move constructor or move assignment operator class value : public x3::variant` – Mankka Nov 23 '17 at 12:48