Template compilation errors with boost::spirit::qi and qi::alnum for C-style string

Question

I use spirit in boost_1.59 to parse a c-like language(named stone originally written by java). But I have trouble in using boost::spirit::qi to parse the c-style string in stone. The whole code is at coliru.

The parser rule I wrote is listed below.

template <typename Iterator, typename Lexer>
struct StoneGrammar
    : qi::grammar<Iterator, qi::in_state_skipper<Lexer> >
{
    template <typename TokenDef>
    StoneGrammar(TokenDef const& tok)
        : StoneGrammar::base_type(program)
    {
        using boost::spirit::_val;
        using boost::spirit::no_skip;

        /* Grammar for Stone:
        primary    : "(" expr ")" | NUMBER | IDENTIFIER | STRING
        factor     : "-" primary | primary
        expr       : factor { OP factor }
        block      : "{" [ statement ] { (";" | EOL) [ statement ] } "}"
        //block      : "{" [ statement ] { (";" | EOL) [ statement ] } "}"
        simple     : expr
        statement  : "if" expr block [ "else" block ]
                   | "while" expr block
                   | simple

        program    : { statement [";"] }
        */
        program
        = -statement >> (qi::lit(';') | '\n')
        ;

        statement
            = if_stmt
            | while_stmt
            | simple_stmt
            ;

        if_stmt
            = "if" >> expression >> block >> -("else" >> block)
            ; 

        while_stmt
            = "while" >> expression >> block;

        simple_stmt
            = expression
            ;

        block
            = '{' >> -statement >> *((qi::lit(';') | '\n') >> -statement) >> '}'
            ;

        expression
            = factor >> *(op >> factor)
            ;

        factor
            = '-' >> primary
            | primary
            ;

        op
            = qi::lit('+')
            | '-'
            | '*'
            | '/'
            | qi::lit('=')[std::cout << _1 << "= parserd" << std::endl]
            | qi::lit("||")[std::cout << _1 << " || parsed" << std::endl]
            ;

        primary
            = '(' >> expression >> ')'
            | tok.identifier[std::cout << _1 << std::endl]
            | tok.number[std::cout << _1 << std::endl]
            | unesc_str
            ;


        unesc_char.add("\\a", '\a')("\\b", '\b')("\\f", '\f')("\\n", '\n')
            ("\\r", '\r')("\\t", '\t')("\\v", '\v')
            ("\\\\", '\\')("\\\'", '\'')("\\\"", '\"')
            ;


        unesc_str = qi::lit('"') >> (qi::alnum | ("\\x" >> qi::hex)) >> '"';
    }

    typedef boost::variant<unsigned int, std::string> expression_type;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > program, block, statement;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > assignment, if_stmt;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > while_stmt;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > simple_stmt;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > op;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > factor;
    qi::rule<Iterator, qi::in_state_skipper<Lexer> > primary;
    qi::rule<Iterator, std::string(), qi::in_state_skipper<Lexer> > string_literal;
    qi::rule<Iterator, std::string(), qi::in_state_skipper<Lexer> > unesc_str;
    qi::symbols<char const, char const> unesc_char;

    //  the expression is the only rule having a return value
    qi::rule<Iterator, expression_type(), qi::in_state_skipper<Lexer> >  expression;
};

The parser use some token definition is listed below.

template <typename Lexer>
struct StoneToken : lex::lexer<Lexer>
{
    StoneToken()
    {
        // define the tokens to match
        identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
        number = "[0-9]+";

        // !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
        // associate the tokens and the token set with the lexer
        this->self
            = lex::token_def<>('(') | ')' | '{' | '}' | '[' | ']'
            | '+' | '-' | '*' | '/'
            | '='
            | "==" | "<=" | ">="
            | '>' | '<' | "&&"
            //| "\||"
            | '!' | '^' | '|' | '~' | '&' 
            | '%' 
            | '\'' | ',' | '"' | '\n' | ';' | '.' | '_'
            | "kkk"
            ;

        //this->self += number | if_ | else_ | while_ | identifier;
        this->self += number | identifier;

        // define the whitespace to ignore (spaces, tabs, and C-style 
        // comments)
        this->self("WS")
            = lex::token_def<>("[ \\t]+")
            | R"(\/\*[^*]*\*+([^/*][^*]*\*+)*\/)"
            | R"(\/\/[^\n]*\n)"
            ;
    }


    lex::token_def<std::string> identifier, op;
    lex::token_def<unsigned int> number;
 };    

However, the compiler cannot compile the unesc_str rule. The beginning part of error report from clang++-3.6 is listed below.

/usr/include/boost/type_traits/make_unsigned.hpp:38:4: error: static_assert failed
      "(::boost::type_traits::ice_or< ::boost::is_integral<T>::value,
      ::boost::is_enum<T>::value>::value)"
   BOOST_STATIC_ASSERT(
   ^
/usr/include/boost/static_assert.hpp:78:41: note: expanded from macro 'BOOST_STATIC_ASSERT'
#     define BOOST_STATIC_ASSERT( ... ) static_assert(__VA_ARGS__, #__VA_ARGS__)
                                        ^
/usr/include/boost/type_traits/make_unsigned.hpp:146:70: note: in instantiation of template class
      'boost::detail::make_unsigned_imp<boost::spirit::lex::lexertl::token<__gnu_cxx::__normal_iterator<char
      *, std::basic_string<char> >, boost::mpl::vector<unsigned int, std::basic_string<char>, mpl_::na,
      mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na,
      mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>,
      mpl_::bool_<true>, unsigned long> >' requested here
BOOST_TT_AUX_TYPE_TRAIT_DEF1(make_unsigned,T,typename boost::detail::make_unsigned_imp<T>::type)
                                                                     ^
/usr/include/boost/type_traits/detail/type_trait_def.hpp:21:13: note: expanded from macro
      'BOOST_TT_AUX_TYPE_TRAIT_DEF1'
    typedef result type; \
            ^
/usr/include/boost/spirit/home/support/char_class.hpp:51:34: note: in instantiation of template class
      'boost::make_unsigned<boost::spirit::lex::lexertl::token<__gnu_cxx::__normal_iterator<char *,
      std::basic_string<char> >, boost::mpl::vector<unsigned int, std::basic_string<char>, mpl_::na,
      mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na,
      mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>,
      mpl_::bool_<true>, unsigned long> >' requested here
                typedef typename make_unsigned<SourceChar>::type USourceChar;

Howerver, I found out that if i deleted the qi::alnum in unesc_str rule, the compilation was successful.

Waiting for answers!

Answer 1

Your problems (mainly) arise from the use of a lexer. Why do have it?

There's a logical contradiction between lexing tokens first, and then parsing a string using a sequence of character parsers.

The most obvious workaround would be to create "tokens" for alnum and whatnot, but frankly I get the impression you are a bit new to spirit and need to start simpler.

If you want I can livecode a simpler grammar later.

---

Side notes: prefer not to use using namespace because you will get spurious (silent) conflict. E.g. it was really hard for me to know whether ::_1 or boost::spirit::qi::_1 was being used there.

Here's half a cleanup, that allowed me to approach the question you were asking: http://paste.ubuntu.com/12724893/

UPDATE

So, I enjoyed myself for faaaaar too long with that pretty impotent grammar :) Here is a nice online demo. I've gathered the livecoding streams here: part #1, part #2, part #3 and part #4

Here's a working grammar that parses those productions:

start                = skip(blank) [ program ];

program              = -statement % (eol|';');
statement            = conditionalstatement | whileloop | simple;

simple               = !eoi >> expression.alias();
conditionalstatement = "if" >> expression >> block >> -("else" >> block);
whileloop            = "while" >> expression >> block;

block                = '{' >> program >> '}';

expression           = e_top.alias();

e_simple             = '(' >> expression >> ')' | number | string | identifier;

e_top                = attr_cast<ast::BinaryExpr, ast::BinaryExpr>(copy(e_factor >> char_("*/") >> e_top))    | e_factor;
e_factor             = attr_cast<ast::BinaryExpr, ast::BinaryExpr>(copy(e_term   >> char_("-+") >> e_factor)) | e_term;
e_term               = unaryexpr | e_simple;

unaryexpr            = char_('-') >> e_simple;
number               = real_parser<double, ureal_policies<double> >();
string               = '"' >> *~char_('"') >> '"';
identifier           = raw [ alpha >> *(alnum | '_') ];

The expression parser has some "artificial" nodes that make sure the AST reflects usual operator precedence.

The ast namespace contains the types exposed. I actually write these first (see the live stream recordings):

namespace {
    template <typename Tag> struct Literal;

    namespace Tags {
        struct NumberLiteral;
        struct StringLiteral;
    }

    template <> struct Literal<Tags::NumberLiteral> { double      value; };
    template <> struct Literal<Tags::StringLiteral> { std::string value; };
}

namespace {
    using Identifier = std::string;
    using Number     = Literal<Tags::NumberLiteral>;
    using String     = Literal<Tags::StringLiteral>;

    struct UnaryExpr;
    struct BinaryExpr;

    using Expression = boost::make_recursive_variant<
            Number,
            Identifier,
            String,
            boost::recursive_variant_, // refers to Expression itself
            boost::recursive_wrapper<UnaryExpr>,
            boost::recursive_wrapper<BinaryExpr>
        >::type;

    struct UnaryExpr {
        char op; // -
        Expression expr;
    };

    struct BinaryExpr {
        char op; // +-/*
        Expression lhs, rhs;
    };

    using Simple = Expression;
}

namespace {
    struct ConditionalStatement;
    struct WhileLoop;

    using Statement  = boost::make_recursive_variant<
            boost::recursive_wrapper<ConditionalStatement>,
            boost::recursive_wrapper<WhileLoop>,
            Simple
        >::type;

    using Block = std::vector<Statement>;

    struct ConditionalStatement {
        Expression             condition;
        Block                  true_branch;
        boost::optional<Block> false_branch;
    };

    struct WhileLoop {
        Expression condition;
        Block      body;
    };
}

using Program = Block;

The code to parse is really simple:

    std::ifstream ifs(fname, std::ios::binary);
    It first(ifs >> std::noskipws), last;

    stone::ast::Program program;
    bool ok = qi::parse(first, last, parser, program);

    if (ok)
    {
        std::cout << "File " << fname << " was parsed succesfully: ";
        dumper(program);
        std::cout << "\n";
    }
    else
        std::cout << "File " << fname << " failed to parse\n";

    if (first!=last)
        std::cout << "Warning: trailing unparsed input '" << std::string(first, last) << "'\n";

For the input test1.stone:

1; if IsValid {
    make
    noise;

    while 2 * (8+1)/"something else that is unimportant" {
        shoot; cannon
    }

} else { cry_in_7_corners_; }

It dumps the AST nodes:

File test1.stone was parsed succesfully: { 1; if [IsValid]{ [make]; [noise]; while (2* ((8+ 1)/ "something else that is unimportant")){ [shoot]; [cannon]; } ; } else { [cry_in_7_corners_]; } ; }